diff --git a/tests/test_helpers.py b/tests/test_helpers.py index a0e4a195..81065cc9 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -9,8 +9,9 @@ import doctest import unittest import warnings +import string -from texthero import _helper +from texthero import helper, preprocessing, nlp """ Doctests. @@ -18,7 +19,7 @@ def load_tests(loader, tests, ignore): - tests.addTests(doctest.DocTestSuite(_helper)) + tests.addTests(doctest.DocTestSuite(helper)) return tests @@ -35,7 +36,7 @@ class TestHelpers(PandasTestCase): def test_handle_nans(self): s = pd.Series(["Test", np.nan, pd.NA]) - @_helper.handle_nans(replace_nans_with="This was a NAN") + @helper.handle_nans(replace_nans_with="This was a NAN") def f(s): return s @@ -51,7 +52,7 @@ def f(s): def test_handle_nans_no_nans_in_input(self): s = pd.Series(["Test"]) - @_helper.handle_nans(replace_nans_with="This was a NAN") + @helper.handle_nans(replace_nans_with="This was a NAN") def f(s): return s @@ -63,7 +64,7 @@ def f(s): def test_handle_nans_index(self): s = pd.Series(["Test", np.nan, pd.NA], index=[4, 5, 6]) - @_helper.handle_nans(replace_nans_with="This was a NAN") + @helper.handle_nans(replace_nans_with="This was a NAN") def f(s): return s @@ -74,3 +75,265 @@ def f(s): with warnings.catch_warnings(): warnings.simplefilter("ignore") self.assertTrue(f(s).index.equals(s_true.index)) + + +class TestPreprocessingParallelized(PandasTestCase): + """ + Test remove digits. + """ + + def setUp(self): + helper.MIN_LINES_FOR_PARALLELIZATION = 0 + helper.PARALLELIZE = True + + def tearDown(self): + helper.MIN_LINES_FOR_PARALLELIZATION = 10000 + helper.PARALLELIZE = True + + def parallelized_test_helper(self, func, s, non_parallel_s_true, **kwargs): + + s = s + non_parallel_s_true = non_parallel_s_true + + pd.testing.assert_series_equal(non_parallel_s_true, func(s, **kwargs)) + + def test_remove_digits_only_block(self): + s = pd.Series("remove block of digits 1234 h1n1") + s_true = pd.Series("remove block of digits h1n1") + self.parallelized_test_helper(preprocessing.remove_digits, s, s_true) + + def test_remove_digits_any(self): + s = pd.Series("remove block of digits 1234 h1n1") + s_true = pd.Series("remove block of digits h n ") + + self.parallelized_test_helper( + preprocessing.remove_digits, s, s_true, only_blocks=False + ) + + def test_remove_digits_brackets(self): + s = pd.Series("Digits in bracket (123 $) needs to be cleaned out") + s_true = pd.Series("Digits in bracket ( $) needs to be cleaned out") + self.parallelized_test_helper(preprocessing.remove_digits, s, s_true) + + def test_remove_digits_start(self): + s = pd.Series("123 starting digits needs to be cleaned out") + s_true = pd.Series(" starting digits needs to be cleaned out") + self.parallelized_test_helper(preprocessing.remove_digits, s, s_true) + + def test_remove_digits_end(self): + s = pd.Series("end digits needs to be cleaned out 123") + s_true = pd.Series("end digits needs to be cleaned out ") + self.parallelized_test_helper(preprocessing.remove_digits, s, s_true) + + def test_remove_digits_phone(self): + s = pd.Series("+41 1234 5678") + s_true = pd.Series("+ ") + self.parallelized_test_helper(preprocessing.remove_digits, s, s_true) + + def test_remove_digits_punctuation(self): + s = pd.Series(string.punctuation) + s_true = pd.Series(string.punctuation) + self.parallelized_test_helper(preprocessing.remove_digits, s, s_true) + + """ + Test replace digits + """ + + def test_replace_digits(self): + s = pd.Series("1234 falcon9") + s_true = pd.Series("X falcon9") + self.parallelized_test_helper( + preprocessing.replace_digits, s, s_true, symbols="X" + ) + + def test_replace_digits_any(self): + s = pd.Series("1234 falcon9") + s_true = pd.Series("X falconX") + self.parallelized_test_helper( + preprocessing.replace_digits, s, s_true, symbols="X", only_blocks=False + ) + + """ + Remove punctuation. + """ + + def test_remove_punctation(self): + s = pd.Series("Remove all! punctuation!! ()") + s_true = pd.Series( + "Remove all punctuation " + ) # TODO maybe just remove space? + self.parallelized_test_helper(preprocessing.remove_punctuation, s, s_true) + + """ + Remove diacritics. + """ + + def test_remove_diactitics(self): + s = pd.Series("Montréal, über, 12.89, Mère, Françoise, noël, 889, اِس, اُس") + s_true = pd.Series("Montreal, uber, 12.89, Mere, Francoise, noel, 889, اس, اس") + self.parallelized_test_helper(preprocessing.remove_diacritics, s, s_true) + + """ + Remove whitespace. + """ + + def test_remove_whitespace(self): + s = pd.Series("hello world hello world ") + s_true = pd.Series("hello world hello world") + self.parallelized_test_helper(preprocessing.remove_whitespace, s, s_true) + + """ + Test pipeline. + """ + + def test_pipeline_stopwords(self): + s = pd.Series("E-I-E-I-O\nAnd on") + s_true = pd.Series("e-i-e-i-o\n ") + pipeline = [preprocessing.lowercase, preprocessing.remove_stopwords] + self.parallelized_test_helper(preprocessing.clean, s, s_true, pipeline=pipeline) + + """ + Test remove html tags + """ + + def test_remove_html_tags(self): + s = pd.Series("remove
html
tags  ") + s_true = pd.Series("remove html tags ") + self.parallelized_test_helper(preprocessing.remove_html_tags, s, s_true) + + """ + Text tokenization + """ + + def test_tokenize(self): + s = pd.Series("text to tokenize") + s_true = pd.Series([["text", "to", "tokenize"]]) + self.parallelized_test_helper(preprocessing.tokenize, s, s_true) + + """ + Has content + """ + + def test_has_content(self): + s = pd.Series(["c", np.nan, "\t\n", " ", "", "has content", None]) + s_true = pd.Series([True, False, False, False, False, True, False]) + self.parallelized_test_helper(preprocessing.has_content, s, s_true) + + """ + Test remove urls + """ + + def test_remove_urls(self): + s = pd.Series("http://tests.com http://www.tests.com") + s_true = pd.Series(" ") + self.parallelized_test_helper(preprocessing.remove_urls, s, s_true) + + """ + Remove brackets + """ + + def test_remove_brackets(self): + s = pd.Series( + "Remove all [square_brackets]{/curly_brackets}(round_brackets)" + ) + s_true = pd.Series("Remove all ") + self.parallelized_test_helper(preprocessing.remove_brackets, s, s_true) + + """ + Test replace and remove tags + """ + + def test_replace_tags(self): + s = pd.Series("Hi @tag, we will replace you") + s_true = pd.Series("Hi TAG, we will replace you") + self.parallelized_test_helper( + preprocessing.replace_tags, s, s_true, symbol="TAG" + ) + + def test_remove_tags_alphabets(self): + s = pd.Series("Hi @tag, we will remove you") + s_true = pd.Series("Hi , we will remove you") + + self.parallelized_test_helper(preprocessing.remove_tags, s, s_true) + + """ + Test replace and remove hashtags + """ + + def test_replace_hashtags(self): + s = pd.Series("Hi #hashtag, we will replace you") + s_true = pd.Series("Hi HASHTAG, we will replace you") + + self.parallelized_test_helper( + preprocessing.replace_hashtags, s, s_true, symbol="HASHTAG" + ) + + def test_remove_hashtags(self): + s = pd.Series("Hi #hashtag_trending123, we will remove you") + s_true = pd.Series("Hi , we will remove you") + + self.parallelized_test_helper(preprocessing.remove_hashtags, s, s_true) + + """ + Test NLP for parallelization + """ + + """ + Named entity. + """ + + def test_named_entities(self): + s = pd.Series("New York is a big city") + s_true = pd.Series([[("New York", "GPE", 0, 8)]]) + self.parallelized_test_helper(nlp.named_entities, s, s_true) + + """ + Noun chunks. + """ + + def test_noun_chunks(self): + s = pd.Series("Today is such a beautiful day") + s_true = pd.Series( + [[("Today", "NP", 0, 5), ("such a beautiful day", "NP", 9, 29)]] + ) + + self.parallelized_test_helper(nlp.noun_chunks, s, s_true) + + """ + Count sentences. + """ + + def test_count_sentences(self): + s = pd.Series("I think ... it counts correctly. Doesn't it? Great!") + s_true = pd.Series(3) + self.parallelized_test_helper(nlp.count_sentences, s, s_true) + + """ + POS tagging. + """ + + def test_pos(self): + s = pd.Series(["Today is such a beautiful day", "São Paulo is a great city"]) + + s_true = pd.Series( + [ + [ + ("Today", "NOUN", "NN", 0, 5), + ("is", "AUX", "VBZ", 6, 8), + ("such", "DET", "PDT", 9, 13), + ("a", "DET", "DT", 14, 15), + ("beautiful", "ADJ", "JJ", 16, 25), + ("day", "NOUN", "NN", 26, 29), + ], + [ + ("São", "PROPN", "NNP", 0, 3), + ("Paulo", "PROPN", "NNP", 4, 9), + ("is", "AUX", "VBZ", 10, 12), + ("a", "DET", "DT", 13, 14), + ("great", "ADJ", "JJ", 15, 20), + ("city", "NOUN", "NN", 21, 25), + ], + ] + ) + + self.parallelized_test_helper(nlp.pos_tag, s, s_true) diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index 4ca3ace2..6cec62b0 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -177,7 +177,7 @@ def test_tokenize_split_punctuation(self): def test_tokenize_not_split_in_between_punctuation(self): s = pd.Series(["don't say hello-world hello_world"]) s_true = pd.Series([["don't", "say", "hello-world", "hello_world"]]) - self.assertEqual(preprocessing.tokenize(s), s_true) + pd.testing.assert_series_equal(preprocessing.tokenize(s), s_true) """ Has content @@ -186,7 +186,7 @@ def test_tokenize_not_split_in_between_punctuation(self): def test_has_content(self): s = pd.Series(["c", np.nan, "\t\n", " ", "", "has content", None]) s_true = pd.Series([True, False, False, False, False, True, False]) - self.assertEqual(preprocessing.has_content(s), s_true) + pd.testing.assert_series_equal(preprocessing.has_content(s), s_true) """ Test remove urls @@ -195,17 +195,17 @@ def test_has_content(self): def test_remove_urls(self): s = pd.Series("http://tests.com http://www.tests.com") s_true = pd.Series(" ") - self.assertEqual(preprocessing.remove_urls(s), s_true) + pd.testing.assert_series_equal(preprocessing.remove_urls(s), s_true) def test_remove_urls_https(self): s = pd.Series("https://tests.com https://www.tests.com") s_true = pd.Series(" ") - self.assertEqual(preprocessing.remove_urls(s), s_true) + pd.testing.assert_series_equal(preprocessing.remove_urls(s), s_true) def test_remove_urls_multiline(self): s = pd.Series("https://tests.com \n https://tests.com") s_true = pd.Series(" \n ") - self.assertEqual(preprocessing.remove_urls(s), s_true) + pd.testing.assert_series_equal(preprocessing.remove_urls(s), s_true) """ Remove brackets diff --git a/texthero/__init__.py b/texthero/__init__.py index 66e891e9..c04fc2ef 100644 --- a/texthero/__init__.py +++ b/texthero/__init__.py @@ -16,3 +16,8 @@ from .nlp import * from . import stopwords + +from . import helper + +from . import config +from .config import * diff --git a/texthero/config.py b/texthero/config.py new file mode 100644 index 00000000..c464d4fa --- /dev/null +++ b/texthero/config.py @@ -0,0 +1,2 @@ +MIN_LINES_FOR_PARALLELIZATION = 10000 +PARALLELIZE = True diff --git a/texthero/_helper.py b/texthero/helper.py similarity index 68% rename from texthero/_helper.py rename to texthero/helper.py index 762dd203..ecf0a653 100644 --- a/texthero/_helper.py +++ b/texthero/helper.py @@ -1,11 +1,15 @@ """ Useful helper functions for the texthero library. """ - +import sys import pandas as pd +import multiprocessing as mp +import numpy as np import functools import warnings +from texthero import config + """ Warnings. @@ -36,7 +40,7 @@ def handle_nans(replace_nans_with): Examples -------- - >>> from texthero._helper import handle_nans + >>> from texthero.helper import handle_nans >>> import pandas as pd >>> import numpy as np >>> @handle_nans(replace_nans_with="I was missing!") @@ -71,3 +75,37 @@ def wrapper(*args, **kwargs): return wrapper return decorator + + +""" +Parallelization. +""" + + +cores = mp.cpu_count() +partitions = cores + + +def parallel(s, func, *args, **kwargs): + + if len(s) < config.MIN_LINES_FOR_PARALLELIZATION or not config.PARALLELIZE: + # Execute as usual. + return func(s, *args, **kwargs) + + else: + # Execute in parallel. + + # Split the data up into batches. + s_split = np.array_split(s, partitions) + + # Open threadpool. + pool = mp.Pool(cores) + # Execute in parallel and concat results (order is kept). + s_result = pd.concat( + pool.map(functools.partial(func, *args, **kwargs), s_split) + ) + + pool.close() + pool.join() + + return s_result diff --git a/texthero/nlp.py b/texthero/nlp.py index 748f0cd8..620c1272 100644 --- a/texthero/nlp.py +++ b/texthero/nlp.py @@ -7,6 +7,19 @@ import en_core_web_sm from nltk.stem import PorterStemmer, SnowballStemmer from texthero._types import TextSeries, InputSeries +from texthero.helper import parallel + + +def _named_entities(s: TextSeries, nlp) -> pd.Series: + + entities = [] + + for doc in nlp.pipe(s.astype("unicode").values, batch_size=32): + entities.append( + [(ent.text, ent.label_, ent.start_char, ent.end_char) for ent in doc.ents] + ) + + return pd.Series(entities, index=s.index) @InputSeries(TextSeries) @@ -66,6 +79,22 @@ def named_entities(s: TextSeries, package="spacy") -> pd.Series: @InputSeries(TextSeries) +def _noun_chunks(s: TextSeries, nlp) -> pd.Series: + + noun_chunks = [] + + # nlp.pipe is now "tagger", "parser" + for doc in nlp.pipe(s.astype("unicode").values, batch_size=32): + noun_chunks.append( + [ + (chunk.text, chunk.label_, chunk.start_char, chunk.end_char) + for chunk in doc.noun_chunks + ] + ) + + return pd.Series(noun_chunks, index=s.index) + + def noun_chunks(s: TextSeries) -> pd.Series: """ Return noun chunks (noun phrases). @@ -91,20 +120,20 @@ def noun_chunks(s: TextSeries) -> pd.Series: dtype: object """ - noun_chunks = [] - nlp = en_core_web_sm.load(disable=["ner"]) - # nlp.pipe is now "tagger", "parser" - for doc in nlp.pipe(s.astype("unicode").values, batch_size=32): - noun_chunks.append( - [ - (chunk.text, chunk.label_, chunk.start_char, chunk.end_char) - for chunk in doc.noun_chunks - ] - ) + return parallel(s, _noun_chunks, nlp=nlp) - return pd.Series(noun_chunks, index=s.index) + +def _count_sentences(s: TextSeries, nlp) -> pd.Series: + + number_of_sentences = [] + + for doc in nlp.pipe(s.values, batch_size=32): + sentences = len(list(doc.sents)) + number_of_sentences.append(sentences) + + return pd.Series(number_of_sentences, index=s.index) @InputSeries(TextSeries) @@ -129,17 +158,27 @@ def count_sentences(s: TextSeries) -> pd.Series: 1 3 dtype: int64 """ - number_of_sentences = [] nlp = en_core_web_sm.load(disable=["tagger", "parser", "ner"]) nlp.add_pipe(nlp.create_pipe("sentencizer")) # Pipe is only "sentencizer" - for doc in nlp.pipe(s.values, batch_size=32): - sentences = len(list(doc.sents)) - number_of_sentences.append(sentences) + return parallel(s, _count_sentences, nlp=nlp) - return pd.Series(number_of_sentences, index=s.index) + +def _pos_tag(s: TextSeries, nlp) -> pd.Series: + + pos_tags = [] + + for doc in nlp.pipe(s.astype("unicode").values, batch_size=32): + pos_tags.append( + [ + (token.text, token.pos_, token.tag_, token.idx, token.idx + len(token)) + for token in doc + ] + ) + + return pd.Series(pos_tags, index=s.index) @InputSeries(TextSeries) @@ -219,6 +258,13 @@ def pos_tag(s: TextSeries) -> pd.Series: return pd.Series(pos_tags, index=s.index) +def _stem(s, stemmer): + def _stem_algorithm(text): + return " ".join([stemmer.stem(word) for word in text]) + + return s.str.split().apply(_stem_algorithm) + + @InputSeries(TextSeries) def stem(s: TextSeries, stem="snowball", language="english") -> TextSeries: r""" @@ -269,7 +315,4 @@ def stem(s: TextSeries, stem="snowball", language="english") -> TextSeries: else: raise ValueError("stem argument must be either 'porter' of 'stemmer'") - def _stem(text): - return " ".join([stemmer.stem(word) for word in text]) - - return s.str.split().apply(_stem) + return parallel(s, _stem, stemmer=stemmer) diff --git a/texthero/preprocessing.py b/texthero/preprocessing.py index 747c9598..8877f210 100644 --- a/texthero/preprocessing.py +++ b/texthero/preprocessing.py @@ -14,6 +14,7 @@ from texthero import stopwords as _stopwords from texthero._types import TokenSeries, TextSeries, InputSeries +from texthero.helper import parallel from typing import List, Callable, Union @@ -23,6 +24,10 @@ warnings.filterwarnings(action="ignore", category=UserWarning, module="gensim") +def _fillna(s: TextSeries) -> TextSeries: + return s.fillna("").astype("str") + + @InputSeries(TextSeries) def fillna(s: TextSeries) -> TextSeries: """ @@ -41,7 +46,11 @@ def fillna(s: TextSeries) -> TextSeries: 3 You're dtype: object """ - return s.fillna("").astype("str") + return parallel(s, _fillna) + + +def _lowercase(s: TextSeries) -> TextSeries: + return s.str.lower() @InputSeries(TextSeries) @@ -59,7 +68,15 @@ def lowercase(s: TextSeries) -> TextSeries: 0 this is new york with upper letters dtype: object """ - return s.str.lower() + return parallel(s, _lowercase) + + +def _replace_digits(s: TextSeries, symbols: str = " ", only_blocks=True) -> TextSeries: + if only_blocks: + pattern = r"\b\d+\b" + return s.str.replace(pattern, symbols) + else: + return s.str.replace(r"\d+", symbols) @InputSeries(TextSeries) @@ -94,12 +111,7 @@ def replace_digits(s: TextSeries, symbols: str = " ", only_blocks=True) -> TextS 0 X falconX dtype: object """ - - if only_blocks: - pattern = r"\b\d+\b" - return s.str.replace(pattern, symbols) - else: - return s.str.replace(r"\d+", symbols) + return parallel(s, _replace_digits, symbols=symbols, only_blocks=only_blocks) @InputSeries(TextSeries) @@ -137,6 +149,10 @@ def remove_digits(s: TextSeries, only_blocks=True) -> TextSeries: return replace_digits(s, " ", only_blocks) +def _replace_punctuation(s: TextSeries, symbol: str = " ") -> TextSeries: + return s.str.replace(rf"([{string.punctuation}])+", symbol) + + @InputSeries(TextSeries) def replace_punctuation(s: TextSeries, symbol: str = " ") -> TextSeries: """ @@ -164,8 +180,7 @@ def replace_punctuation(s: TextSeries, symbol: str = " ") -> TextSeries: 0 Finnaly dtype: object """ - - return s.str.replace(rf"([{string.punctuation}])+", symbol) + return parallel(s, _replace_punctuation, symbol=symbol) @InputSeries(TextSeries) @@ -192,7 +207,7 @@ def remove_punctuation(s: TextSeries) -> TextSeries: return replace_punctuation(s, " ") -def _remove_diacritics(text: str) -> str: +def _remove_diacritics_algorithm(text: str) -> str: """ Remove diacritics and accents from one string. @@ -201,15 +216,20 @@ def _remove_diacritics(text: str) -> str: >>> from texthero.preprocessing import _remove_diacritics >>> import pandas as pd >>> text = "Montréal, über, 12.89, Mère, Françoise, noël, 889, اِس, اُس" - >>> _remove_diacritics(text) + >>> _remove_diacritics_algorithm(text) 'Montreal, uber, 12.89, Mere, Francoise, noel, 889, اس, اس' """ + nfkd_form = unicodedata.normalize("NFKD", text) # unicodedata.combining(char) checks if the character is in # composed form (consisting of several unicode chars combined), i.e. a diacritic return "".join([char for char in nfkd_form if not unicodedata.combining(char)]) +def _remove_diacritics(s: TextSeries) -> TextSeries: + return s.astype("unicode").apply(_remove_diacritics_algorithm) + + @InputSeries(TextSeries) def remove_diacritics(s: TextSeries) -> TextSeries: """ @@ -229,7 +249,11 @@ def remove_diacritics(s: TextSeries) -> TextSeries: 'Montreal, uber, 12.89, Mere, Francoise, noel, 889, اس, اس' """ - return s.astype("unicode").apply(_remove_diacritics) + return parallel(s, _remove_diacritics) + + +def _remove_whitespace(s: TextSeries) -> TextSeries: + return s.str.replace("\xa0", " ").str.split().str.join(" ") @InputSeries(TextSeries) @@ -252,11 +276,10 @@ def remove_whitespace(s: TextSeries) -> TextSeries: 0 Title Subtitle ... dtype: object """ - - return s.str.replace("\xa0", " ").str.split().str.join(" ") + return parallel(s, _remove_whitespace) -def _replace_stopwords(text: str, words: Set[str], symbol: str = " ") -> str: +def _replace_stopwords_algorithm(text: str, words: Set[str], symbol: str = " ") -> str: """ Remove words in a set from a string, replacing them with a symbol. @@ -276,7 +299,7 @@ def _replace_stopwords(text: str, words: Set[str], symbol: str = " ") -> str: >>> s = "the book of the jungle" >>> symbol = "$" >>> stopwords = ["the", "of"] - >>> _replace_stopwords(s, stopwords, symbol) + >>> _replace_stopwords_algorithm(s, stopwords, symbol) '$ book $ $ jungle' """ @@ -290,6 +313,12 @@ def _replace_stopwords(text: str, words: Set[str], symbol: str = " ") -> str: return "".join(t if t not in words else symbol for t in re.findall(pattern, text)) +def _replace_stopwords( + s: TextSeries, symbol: str, stopwords: Optional[Set[str]] = None +) -> TextSeries: + return s.apply(_replace_stopwords_algorithm, words=stopwords, symbol=symbol) + + @InputSeries(TextSeries) def replace_stopwords( s: TextSeries, symbol: str, stopwords: Optional[Set[str]] = None @@ -320,10 +349,9 @@ def replace_stopwords( dtype: object """ - if stopwords is None: stopwords = _stopwords.DEFAULT - return s.apply(_replace_stopwords, args=(stopwords, symbol)) + return parallel(s, _replace_stopwords, symbol=symbol, stopwords=stopwords) @InputSeries(TextSeries) @@ -485,6 +513,10 @@ def drop_no_content(s: TextSeries) -> TextSeries: return s[has_content(s)] +def _remove_round_brackets(s: TextSeries) -> TextSeries: + return s.str.replace(r"\([^()]*\)", "") + + @InputSeries(TextSeries) def remove_round_brackets(s: TextSeries) -> TextSeries: """ @@ -508,7 +540,11 @@ def remove_round_brackets(s: TextSeries) -> TextSeries: :meth:`remove_square_brackets` """ - return s.str.replace(r"\([^()]*\)", "") + return parallel(s, _remove_round_brackets) + + +def _remove_curly_brackets(s: TextSeries) -> TextSeries: + return s.str.replace(r"\{[^{}]*\}", "") @InputSeries(TextSeries) @@ -534,7 +570,11 @@ def remove_curly_brackets(s: TextSeries) -> TextSeries: :meth:`remove_square_brackets` """ - return s.str.replace(r"\{[^{}]*\}", "") + return parallel(s, _remove_curly_brackets) + + +def _remove_square_brackets(s: TextSeries) -> TextSeries: + return s.str.replace(r"\[[^\[\]]*\]", "") @InputSeries(TextSeries) @@ -559,9 +599,12 @@ def remove_square_brackets(s: TextSeries) -> TextSeries: :meth:`remove_round_brackets` :meth:`remove_curly_brackets` - """ - return s.str.replace(r"\[[^\[\]]*\]", "") + return parallel(s, _remove_square_brackets) + + +def _remove_angle_brackets(s: TextSeries) -> TextSeries: + return s.str.replace(r"<[^<>]*>", "") @InputSeries(TextSeries) @@ -587,7 +630,7 @@ def remove_angle_brackets(s: TextSeries) -> TextSeries: :meth:`remove_square_brackets` """ - return s.str.replace(r"<[^<>]*>", "") + return parallel(s, _remove_angle_brackets) @InputSeries(TextSeries) @@ -623,6 +666,16 @@ def remove_brackets(s: TextSeries) -> TextSeries: ) +def _remove_html_tags(s: TextSeries) -> TextSeries: + + pattern = r"""(?x) # Turn on free-spacing + <[^>]+> # Remove tags + | &([a-z0-9]+|\#[0-9]{1,6}|\#x[0-9a-f]{1,6}); # Remove   + """ + + return s.str.replace(pattern, "") + + @InputSeries(TextSeries) def remove_html_tags(s: TextSeries) -> TextSeries: """ @@ -642,13 +695,18 @@ def remove_html_tags(s: TextSeries) -> TextSeries: dtype: object """ + return parallel(s, _remove_html_tags) - pattern = r"""(?x) # Turn on free-spacing - <[^>]+> # Remove tags - | &([a-z0-9]+|\#[0-9]{1,6}|\#x[0-9a-f]{1,6}); # Remove   - """ - return s.str.replace(pattern, "") +def _tokenize(s: TextSeries) -> TokenSeries: + punct = string.punctuation.replace("_", "") + # In regex, the metacharacter 'w' is "a-z, A-Z, 0-9, including the _ (underscore) + # character." We therefore remove it from the punctuation string as this is already + # included in \w. + + pattern = rf"((\w)([{punct}])(?:\B|$)|(?:^|\B)([{punct}])(\w))" + + return s.str.replace(pattern, r"\2 \3 \4 \5").str.split() @InputSeries(TextSeries) @@ -673,14 +731,7 @@ def tokenize(s: TextSeries) -> TokenSeries: """ - punct = string.punctuation.replace("_", "") - # In regex, the metacharacter 'w' is "a-z, A-Z, 0-9, including the _ (underscore) - # character." We therefore remove it from the punctuation string as this is already - # included in \w. - - pattern = rf"((\w)([{punct}])(?:\B|$)|(?:^|\B)([{punct}])(\w))" - - return s.str.replace(pattern, r"\2 \3 \4 \5").str.split() + return parallel(s, _tokenize) # Warning message for not-tokenized inputs @@ -745,6 +796,11 @@ def phrases( return pd.Series(phrases.fit_transform(s.values), index=s.index) +def _replace_urls(s: TextSeries, symbol: str) -> TextSeries: + pattern = r"http\S+" + return s.str.replace(pattern, symbol) + + @InputSeries(TextSeries) def replace_urls(s: TextSeries, symbol: str) -> TextSeries: r"""Replace all urls with the given symbol. @@ -772,10 +828,7 @@ def replace_urls(s: TextSeries, symbol: str) -> TextSeries: :meth:`texthero.preprocessing.remove_urls` """ - - pattern = r"http\S+" - - return s.str.replace(pattern, symbol) + return parallel(s, _replace_urls, symbol=symbol) @InputSeries(TextSeries) @@ -802,6 +855,12 @@ def remove_urls(s: TextSeries) -> TextSeries: return replace_urls(s, " ") +@InputSeries(TextSeries) +def _replace_tags(s: TextSeries, symbol: str) -> TextSeries: + pattern = r"@[a-zA-Z0-9]+" + return s.str.replace(pattern, symbol) + + @InputSeries(TextSeries) def replace_tags(s: TextSeries, symbol: str) -> TextSeries: """Replace all tags from a given Pandas Series with symbol. @@ -826,9 +885,7 @@ def replace_tags(s: TextSeries, symbol: str) -> TextSeries: dtype: object """ - - pattern = r"@[a-zA-Z0-9]+" - return s.str.replace(pattern, symbol) + return parallel(s, _replace_tags, symbol=symbol) @InputSeries(TextSeries) @@ -856,6 +913,11 @@ def remove_tags(s: TextSeries) -> TextSeries: return replace_tags(s, " ") +def _replace_hashtags(s: TextSeries, symbol: str) -> TextSeries: + pattern = r"#[a-zA-Z0-9_]+" + return s.str.replace(pattern, symbol) + + @InputSeries(TextSeries) def replace_hashtags(s: TextSeries, symbol: str) -> TextSeries: """Replace all hashtags from a Pandas Series with symbol @@ -880,8 +942,7 @@ def replace_hashtags(s: TextSeries, symbol: str) -> TextSeries: dtype: object """ - pattern = r"#[a-zA-Z0-9_]+" - return s.str.replace(pattern, symbol) + return parallel(s, _replace_hashtags, symbol=symbol) @InputSeries(TextSeries)