From fa342a92d4f007cebfce29f1f22a4e31fedc56c6 Mon Sep 17 00:00:00 2001 From: Maximilian Krahn Date: Tue, 18 Aug 2020 22:06:14 +0200 Subject: [PATCH 01/23] added MultiIndex DF support suport MultiIndex as function parameter returns MultiIndex, where Representation was returned * missing: correct test Co-authored-by: Henri Froese --- tests/test_indexes.py | 18 +-- tests/test_representation.py | 63 +------- texthero/representation.py | 294 +++++++++++++---------------------- texthero/visualization.py | 4 +- 4 files changed, 115 insertions(+), 264 deletions(-) diff --git a/tests/test_indexes.py b/tests/test_indexes.py index cc041c3a..af7afcd2 100644 --- a/tests/test_indexes.py +++ b/tests/test_indexes.py @@ -56,21 +56,9 @@ ] test_cases_representation = [ - [ - "count", - lambda x: representation.flatten(representation.count(x)), - (s_tokenized_lists,), - ], - [ - "term_frequency", - lambda x: representation.flatten(representation.term_frequency(x)), - (s_tokenized_lists,), - ], - [ - "tfidf", - lambda x: representation.flatten(representation.tfidf(x)), - (s_tokenized_lists,), - ], + ["count", representation.count, (s_tokenized_lists,),], + ["term_frequency", representation.term_frequency, (s_tokenized_lists,),], + ["tfidf", representation.tfidf, (s_tokenized_lists,),], ["pca", representation.pca, (s_numeric_lists, 0)], ["nmf", representation.nmf, (s_numeric_lists,)], ["tsne", representation.tsne, (s_numeric_lists,)], diff --git a/tests/test_representation.py b/tests/test_representation.py index 036775af..41b81ffa 100644 --- a/tests/test_representation.py +++ b/tests/test_representation.py @@ -50,16 +50,9 @@ def _tfidf(term, corpus, document_index): [["Test", "Test", "TEST", "!"], ["Test", "?", ".", "."]], index=[5, 7] ) -s_tokenized_output_index = pd.MultiIndex.from_tuples( - [(0, "!"), (0, "TEST"), (0, "Test"), (1, "."), (1, "?"), (1, "Test")], -) - -s_tokenized_output_noncontinuous_index = pd.MultiIndex.from_tuples( - [(5, "!"), (5, "TEST"), (5, "Test"), (7, "."), (7, "?"), (7, "Test")], -) - -s_tokenized_output_min_df_index = pd.MultiIndex.from_tuples([(0, "Test"), (1, "Test")],) +s_tokenized_output_index = [0,1] +s_tokenized_output_index_noncontinous = [5,7] test_cases_vectorization = [ # format: [function_name, function, correct output for tokenized input above, dtype of output] @@ -182,55 +175,3 @@ def test_tfidf_formula(self): ).astype("Sparse") self.assertEqual(representation.tfidf(s), s_true) - - """ - flatten. - """ - - def test_flatten(self): - index = pd.MultiIndex.from_tuples( - [("doc0", "Word1"), ("doc0", "Word3"), ("doc1", "Word2")], - ) - s = pd.Series([3, np.nan, 4], index=index) - - s_true = pd.Series( - [[3.0, 0.0, np.nan], [0.0, 4.0, 0.0]], index=["doc0", "doc1"], - ) - - pd.testing.assert_series_equal( - representation.flatten(s), s_true, check_names=False - ) - - def test_flatten_fill_missing_with(self): - index = pd.MultiIndex.from_tuples( - [("doc0", "Word1"), ("doc0", "Word3"), ("doc1", "Word2")], - ) - s = pd.Series([3, np.nan, 4], index=index) - - s_true = pd.Series( - [[3.0, "FILLED", np.nan], ["FILLED", 4.0, "FILLED"]], - index=["doc0", "doc1"], - ) - - pd.testing.assert_series_equal( - representation.flatten(s, fill_missing_with="FILLED"), - s_true, - check_names=False, - ) - - def test_flatten_missing_row(self): - # Simulating a row with no features, so it's completely missing from - # the representation series. - index = pd.MultiIndex.from_tuples( - [("doc0", "Word1"), ("doc0", "Word3"), ("doc1", "Word2")], - ) - s = pd.Series([3, np.nan, 4], index=index) - - s_true = pd.Series( - [[3.0, 0.0, np.nan], [0.0, 4.0, 0.0], [0.0, 0.0, 0.0]], - index=["doc0", "doc1", "doc2"], - ) - - pd.testing.assert_series_equal( - representation.flatten(s, index=s_true.index), s_true, check_names=False - ) diff --git a/texthero/representation.py b/texthero/representation.py index 07b7706c..042db71a 100644 --- a/texthero/representation.py +++ b/texthero/representation.py @@ -27,90 +27,14 @@ """ -def flatten( - s: Union[pd.Series, pd.Series.sparse], - index: pd.Index = None, - fill_missing_with: Any = 0.0, -) -> pd.Series: - """ - Transform a Pandas Representation Series to a "normal" (flattened) Pandas Series. - - The given Series should have a multiindex with first level being the document - and second level being individual features of that document (e.g. tdidf scores per word). - The flattened Series has one cell per document, with the cell being a list of all - the individual features of that document. - - Parameters - ---------- - s : Sparse Pandas Series or Pandas Series - The multiindexed Pandas Series to flatten. - - index : Pandas Index, optional, default to None - The index the flattened Series should have. - - fill_missing_with : Any, default to 0.0 - Value to fill the NaNs (missing values) with. This _does not_ mean - that existing values that are np.nan are replaced, but rather that - features that are not present in one document but present in others - are filled with fill_missing_with. See example below. - - - Examples - -------- - >>> import texthero as hero - >>> import pandas as pd - >>> import numpy as np - >>> index = pd.MultiIndex.from_tuples([("doc0", "Word1"), ("doc0", "Word3"), ("doc1", "Word2")], names=['document', 'word']) - >>> s = pd.Series([3, np.nan, 4], index=index) - >>> s - document word - doc0 Word1 3.0 - Word3 NaN - doc1 Word2 4.0 - dtype: float64 - >>> hero.flatten(s, fill_missing_with=0.0) - document - doc0 [3.0, 0.0, nan] - doc1 [0.0, 4.0, 0.0] - dtype: object - - """ - s = s.unstack(fill_value=fill_missing_with) - - if index is not None: - s = s.reindex(index, fill_value=fill_missing_with) - # Reindexing makes the documents for which no values - # are present in the Sparse Representation Series - # "reappear" correctly. - - s = pd.Series(s.values.tolist(), index=s.index) - - return s - - -def _check_is_valid_representation(s: pd.Series) -> bool: +def _check_is_valid_DocumentTermDF(df: Union[pd.DataFrame, pd.Series]) -> bool: """ - Check if the given Pandas Series is a Document Representation Series. + Check if the given Pandas Series is a Document Term DF. - Returns true if Series is Document Representation Series, else False. + Returns true if input is Document Term DF, else False. """ - - # TODO: in Version 2 when only representation is accepted as input -> change "return False" to "raise ValueError" - - if not isinstance(s.index, pd.MultiIndex): - return False - # raise ValueError( - # f"The input Pandas Series should be a Representation Pandas Series and should have a MultiIndex. The given Pandas Series does not appears to have MultiIndex" - # ) - - if s.index.nlevels != 2: - return False - # raise ValueError( - # f"The input Pandas Series should be a Representation Pandas Series and should have a MultiIndex, where the first level represent the document and the second one the words/token. The given Pandas Series has {s.index.nlevels} number of levels instead of 2." - # ) - - return True + return isinstance(df, pd.DataFrame) and isinstance(df.columns, pd.MultiIndex) # Warning message for not-tokenized inputs @@ -132,11 +56,11 @@ def count( min_df=1, max_df=1.0, binary=False, -) -> pd.Series: +) -> pd.DataFrame: """ Represent a text-based Pandas Series using count. - Return a Document Representation Series with the + Return a Document Term DataFrame with the number of occurences of a document's words for every document. TODO add tutorial link @@ -144,10 +68,6 @@ def count( The input Series should already be tokenized. If not, it will be tokenized before count is calculated. - Use :meth:`hero.representation.flatten` on the output to get - a standard Pandas Series with the document vectors - in every cell. - Parameters ---------- s : Pandas Series (tokenized) @@ -177,15 +97,14 @@ def count( >>> import pandas as pd >>> s = pd.Series(["Sentence one", "Sentence two"]).pipe(hero.tokenize) >>> hero.count(s) - 0 Sentence 1 - one 1 - 1 Sentence 1 - two 1 - dtype: Sparse[int64, 0] + count + Sentence one two + 0 1 1 0 + 1 1 0 1 See Also -------- - Document Representation Series: TODO add tutorial link + Document Term DataFrame: TODO add tutorial link """ # TODO. Can be rewritten without sklearn. @@ -204,25 +123,23 @@ def count( ) tf_vectors_csr = tf.fit_transform(s) - tf_vectors_coo = coo_matrix(tf_vectors_csr) - s_out = pd.Series.sparse.from_coo(tf_vectors_coo) - - features_names = tf.get_feature_names() - - # Map word index to word name - s_out.index = s_out.index.map(lambda x: (s.index[x[0]], features_names[x[1]])) + multiindexed_columns = pd.MultiIndex.from_tuples( + [("count", word) for word in tf.get_feature_names()] + ) - return s_out + return pd.DataFrame.sparse.from_spmatrix( + tf_vectors_csr, s.index, multiindexed_columns + ) def term_frequency( s: pd.Series, max_features: Optional[int] = None, min_df=1, max_df=1.0, -) -> pd.Series: +) -> pd.DataFrame: """ Represent a text-based Pandas Series using term frequency. - Return a Document Representation Series with the + Return a Document Term DataFrame with the term frequencies of the terms for every document. TODO add tutorial link @@ -230,11 +147,6 @@ def term_frequency( The input Series should already be tokenized. If not, it will be tokenized before term_frequency is calculated. - Use :meth:`hero.representation.flatten` on the output to get - a standard Pandas Series with the document vectors - in every cell. - - Parameters ---------- s : Pandas Series (tokenized) @@ -261,16 +173,14 @@ def term_frequency( >>> import pandas as pd >>> s = pd.Series(["Sentence one hey", "Sentence two"]).pipe(hero.tokenize) >>> hero.term_frequency(s) - 0 Sentence 0.2 - hey 0.2 - one 0.2 - 1 Sentence 0.2 - two 0.2 - dtype: Sparse[float64, nan] + term_frequency + Sentence hey one two + 0 0.2 0.2 0.2 0.0 + 1 0.2 0.0 0.0 0.2 See Also -------- - Document Representation Series: TODO add tutorial link + Document Term DataFrame: TODO add tutorial link """ # Check if input is tokenized. Else, print warning and tokenize. if not isinstance(s.iloc[0], list): @@ -291,17 +201,16 @@ def term_frequency( total_count_coo = np.sum(tf_vectors_coo) frequency_coo = np.divide(tf_vectors_coo, total_count_coo) - s_out = pd.Series.sparse.from_coo(frequency_coo) - - features_names = tf.get_feature_names() - - # Map word index to word name - s_out.index = s_out.index.map(lambda x: (s.index[x[0]], features_names[x[1]])) + multiindexed_columns = pd.MultiIndex.from_tuples( + [("term_frequency", word) for word in tf.get_feature_names()] + ) - return s_out + return pd.DataFrame.sparse.from_spmatrix( + frequency_coo, s.index, multiindexed_columns + ) -def tfidf(s: pd.Series, max_features=None, min_df=1, max_df=1.0,) -> pd.Series: +def tfidf(s: pd.Series, max_features=None, min_df=1, max_df=1.0,) -> pd.DataFrame: """ Represent a text-based Pandas Series using TF-IDF. @@ -324,20 +233,13 @@ def tfidf(s: pd.Series, max_features=None, min_df=1, max_df=1.0,) -> pd.Series: so the result is exactly what you get applying the formula described above. - Return a Document Representation Series with the + Return a Document Term DataFrame with the tfidf of every word in the document. TODO add tutorial link The input Series should already be tokenized. If not, it will be tokenized before tfidf is calculated. - If working with big pandas Series, you might want to limit - the number of features through the max_features parameter. - - Use :meth:`hero.representation.flatten` on the output to get - a standard Pandas Series with the document vectors - in every cell. - Parameters ---------- s : Pandas Series (tokenized) @@ -365,17 +267,16 @@ def tfidf(s: pd.Series, max_features=None, min_df=1, max_df=1.0,) -> pd.Series: >>> import pandas as pd >>> s = pd.Series(["Hi Bye", "Test Bye Bye"]).pipe(hero.tokenize) >>> hero.tfidf(s) - 0 Bye 1.000000 - Hi 1.405465 - 1 Bye 2.000000 - Test 1.405465 - dtype: Sparse[float64, nan] + tfidf + Bye Hi Test + 0 1.0 1.405465 0.000000 + 1 2.0 0.000000 1.405465 See Also -------- `TF-IDF on Wikipedia `_ - Document Representation Series: TODO add tutorial link + Document Term DataFrame: TODO add tutorial link """ # Check if input is tokenized. Else, print warning and tokenize. @@ -395,16 +296,13 @@ def tfidf(s: pd.Series, max_features=None, min_df=1, max_df=1.0,) -> pd.Series: tfidf_vectors_csr = tfidf.fit_transform(s) - # Result from sklearn is in Compressed Sparse Row format. - # Pandas Sparse Series can only be initialized from Coordinate format. - tfidf_vectors_coo = coo_matrix(tfidf_vectors_csr) - s_out = pd.Series.sparse.from_coo(tfidf_vectors_coo) - - # Map word index to word name and keep original index of documents. - feature_names = tfidf.get_feature_names() - s_out.index = s_out.index.map(lambda x: (s.index[x[0]], feature_names[x[1]])) + multiindexed_columns = pd.MultiIndex.from_tuples( + [("tfidf", word) for word in tfidf.get_feature_names()] + ) - return s_out + return pd.DataFrame.sparse.from_spmatrix( + tfidf_vectors_csr, s.index, multiindexed_columns + ) """ @@ -412,7 +310,9 @@ def tfidf(s: pd.Series, max_features=None, min_df=1, max_df=1.0,) -> pd.Series: """ -def pca(s, n_components=2, random_state=None) -> pd.Series: +def pca( + s: Union[pd.Series, pd.DataFrame], n_components=2, random_state=None +) -> pd.Series: """ Perform principal component analysis on the given Pandas Series. @@ -434,7 +334,7 @@ def pca(s, n_components=2, random_state=None) -> pd.Series: Parameters ---------- - s : Pandas Series + s : Pandas Series or MuliIndex Sparse DataFrame n_components : Int. Default is 2. Number of components to keep (dimensionality of output vectors). @@ -468,10 +368,18 @@ def pca(s, n_components=2, random_state=None) -> pd.Series: """ pca = PCA(n_components=n_components, random_state=random_state, copy=False) - return pd.Series(pca.fit_transform(list(s)).tolist(), index=s.index) + + if _check_is_valid_DocumentTermDF(s): + values = s.values + else: + values = list(s) + + return pd.Series(pca.fit_transform(values).tolist(), index=s.index) -def nmf(s, n_components=2, random_state=None) -> pd.Series: +def nmf( + s: Union[pd.Series, pd.DataFrame], n_components=2, random_state=None +) -> pd.Series: """ Performs non-negative matrix factorization. @@ -491,7 +399,7 @@ def nmf(s, n_components=2, random_state=None) -> pd.Series: Parameters ---------- - s : Pandas Series + s : Pandas Series or Pandas MultiIndex Sparse DataFrame n_components : Int. Default is 2. Number of components to keep (dimensionality of output vectors). @@ -527,11 +435,17 @@ def nmf(s, n_components=2, random_state=None) -> pd.Series: """ nmf = NMF(n_components=n_components, init="random", random_state=random_state,) - return pd.Series(nmf.fit_transform(list(s)).tolist(), index=s.index) + + if _check_is_valid_DocumentTermDF(s): + values = s.sparse.to_coo() + else: + values = list(s) + + return pd.Series(nmf.fit_transform(values).tolist(), index=s.index) def tsne( - s: pd.Series, + s: Union[pd.Series, pd.DataFrame], n_components=2, perplexity=30.0, learning_rate=200.0, @@ -557,7 +471,7 @@ def tsne( Parameters ---------- - s : Pandas Series + s : Pandas Series or Pandas MultiIndex Sparse DataFrame n_components : int, default is 2. Number of components to keep (dimensionality of output vectors). @@ -619,7 +533,13 @@ def tsne( random_state=random_state, n_jobs=n_jobs, ) - return pd.Series(tsne.fit_transform(list(s)).tolist(), index=s.index) + + if _check_is_valid_DocumentTermDF(s): + values = s.sparse.to_coo() + else: + values = list(s) + + return pd.Series(tsne.fit_transform(values).tolist(), index=s.index) """ @@ -628,7 +548,7 @@ def tsne( def kmeans( - s: pd.Series, + s: Union[pd.Series, pd.DataFrame], n_clusters=5, n_init=10, max_iter=300, @@ -653,7 +573,7 @@ def kmeans( Parameters ---------- - s: Pandas Series + s: Pandas Series or Pandas MultiIndex Sparse DataFrame n_clusters: Int, default to 5. The number of clusters to separate the data into. @@ -686,7 +606,7 @@ def kmeans( >>> import texthero as hero >>> import pandas as pd >>> s = pd.Series(["Football, Sports, Soccer", "music, violin, orchestra", "football, fun, sports", "music, fun, guitar"]) - >>> s = s.pipe(hero.clean).pipe(hero.tokenize).pipe(hero.term_frequency).pipe(hero.flatten) # TODO: when others get Representation Support: remove flatten + >>> s = s.pipe(hero.clean).pipe(hero.tokenize).pipe(hero.term_frequency) >>> hero.kmeans(s, n_clusters=2, random_state=42) 0 1 1 0 @@ -702,7 +622,12 @@ def kmeans( `kmeans on Wikipedia `_ """ - vectors = list(s) + + if _check_is_valid_DocumentTermDF(s): + vectors = s.sparse.to_coo() + else: + vectors = list(s) + kmeans = KMeans( n_clusters=n_clusters, n_init=n_init, @@ -715,7 +640,7 @@ def kmeans( def dbscan( - s, + s: Union[pd.Series, pd.DataFrame], eps=0.5, min_samples=5, metric="euclidean", @@ -743,7 +668,7 @@ def dbscan( Parameters ---------- - s: Pandas Series + s: Pandas Series or Pandas MultiIndex Sparse DataFrame eps : float, default=0.5 The maximum distance between two samples for one to be considered @@ -783,7 +708,7 @@ def dbscan( >>> import texthero as hero >>> import pandas as pd >>> s = pd.Series(["Football, Sports, Soccer", "music, violin, orchestra", "football, fun, sports", "music, enjoy, guitar"]) - >>> s = s.pipe(hero.clean).pipe(hero.tokenize).pipe(hero.tfidf).pipe(hero.flatten) # TODO: when others get Representation Support: remove flatten + >>> s = s.pipe(hero.clean).pipe(hero.tokenize).pipe(hero.tfidf) >>> hero.dbscan(s, min_samples=1, eps=4) 0 0 1 1 @@ -801,6 +726,11 @@ def dbscan( """ + if _check_is_valid_DocumentTermDF(s): + vectors = s.sparse.to_coo() + else: + vectors = list(s) + return pd.Series( DBSCAN( eps=eps, @@ -809,13 +739,13 @@ def dbscan( metric_params=metric_params, leaf_size=leaf_size, n_jobs=n_jobs, - ).fit_predict(list(s)), + ).fit_predict(vectors), index=s.index, ).astype("category") def meanshift( - s, + s: Union[pd.Series, pd.DataFrame], bandwidth=None, bin_seeding=False, min_bin_freq=1, @@ -843,7 +773,7 @@ def meanshift( Parameters ---------- - s: Pandas Series + s: Pandas Series or Pandas MultiIndex Sparse DataFrame bandwidth : float, default=None Bandwidth used in the RBF kernel. @@ -901,6 +831,11 @@ def meanshift( """ + if _check_is_valid_DocumentTermDF(s): + vectors = s.values + else: + vectors = list(s) + return pd.Series( MeanShift( bandwidth=bandwidth, @@ -909,7 +844,7 @@ def meanshift( cluster_all=cluster_all, n_jobs=n_jobs, max_iter=max_iter, - ).fit_predict(list(s)), + ).fit_predict(vectors), index=s.index, ).astype("category") @@ -962,31 +897,18 @@ def normalize(s: pd.Series, norm="l2") -> pd.Series: `Norm on Wikipedia `_ """ + isDocumentTermDF = _check_is_valid_DocumentTermDF(s) - is_valid_representation = ( - isinstance(s.index, pd.MultiIndex) and s.index.nlevels == 2 - ) - - if not is_valid_representation: - raise TypeError( - "The input Pandas Series should be a Representation Pandas Series and should have a MultiIndex. The given Pandas Series does not appears to have MultiIndex" - ) - # TODO after merging representation: use _check_is_valid_representation instead - - if pd.api.types.is_sparse(s): - s_coo_matrix = s.sparse.to_coo()[0] + if isDocumentTermDF: + s_for_vectorization = s.sparse.to_coo() else: - s = s.astype("Sparse") - s_coo_matrix = s.sparse.to_coo()[0] - - s_for_vectorization = s_coo_matrix + s_for_vectorization = list(s) result = sklearn_normalize( s_for_vectorization, norm=norm ) # Can handle sparse input. - result_coo = coo_matrix(result) - s_result = pd.Series.sparse.from_coo(result_coo) - s_result.index = s.index - - return s_result + if isDocumentTermDF: + return pd.DataFrame.sparse.from_spmatrix(result, s.index, s.columns) + else: + return pd.Series(result.tolist(), index=s.index) diff --git a/texthero/visualization.py b/texthero/visualization.py index e213285e..2426ab4d 100644 --- a/texthero/visualization.py +++ b/texthero/visualization.py @@ -63,8 +63,8 @@ def scatterplot( >>> import pandas as pd >>> df = pd.DataFrame(["Football, Sports, Soccer", "music, violin, orchestra", "football, fun, sports", "music, fun, guitar"], columns=["texts"]) >>> df["texts"] = hero.clean(df["texts"]).pipe(hero.tokenize) - >>> df["pca"] = hero.tfidf(df["texts"]).pipe(hero.flatten).pipe(hero.pca, n_components=3) # TODO: when others get Representation Support: remove flatten - >>> df["topics"] = hero.tfidf(df["texts"]).pipe(hero.flatten).pipe(hero.kmeans, n_clusters=2) # TODO: when others get Representation Support: remove flatten + >>> df["pca"] = hero.tfidf(df["texts"]).pipe(hero.pca, n_components=3) + >>> df["topics"] = hero.tfidf(df["texts"]).pipe(hero.kmeans, n_clusters=2) >>> hero.scatterplot(df, col="pca", color="topics", hover_data=["texts"]) # doctest: +SKIP """ From 59a9f8c0df70d8136780b3160bc1d2ca59f48b26 Mon Sep 17 00:00:00 2001 From: Henri Froese Date: Wed, 19 Aug 2020 19:39:30 +0200 Subject: [PATCH 02/23] beginning with tests --- tests/test_representation.py | 147 +++++++++++++++++------------------ texthero/representation.py | 8 +- 2 files changed, 76 insertions(+), 79 deletions(-) diff --git a/tests/test_representation.py b/tests/test_representation.py index 41b81ffa..d4acd369 100644 --- a/tests/test_representation.py +++ b/tests/test_representation.py @@ -50,32 +50,84 @@ def _tfidf(term, corpus, document_index): [["Test", "Test", "TEST", "!"], ["Test", "?", ".", "."]], index=[5, 7] ) -s_tokenized_output_index = [0,1] +s_tokenized_output_index = [0, 1] + +s_tokenized_output_index_noncontinous = [5, 7] + + +def _get_multiindex_for_tokenized_output(first_level_name): + return pd.MultiIndex.from_product( + [[first_level_name], ["!", ".", "?", "TEST", "Test"]] + ) -s_tokenized_output_index_noncontinous = [5,7] test_cases_vectorization = [ - # format: [function_name, function, correct output for tokenized input above, dtype of output] - ["count", representation.count, [1, 1, 2, 2, 1, 1], "int"], + # format: [function_name, function, correct output for tokenized input above] + [ + "count", + representation.count, + pd.DataFrame( + [[1, 0, 0, 1, 2], [0, 2, 1, 0, 1]], + index=s_tokenized_output_index, + columns=_get_multiindex_for_tokenized_output("count"), + ).astype("Sparse"), + ], [ "term_frequency", representation.term_frequency, - [0.125, 0.125, 0.250, 0.250, 0.125, 0.125], - "float", + pd.DataFrame( + [[0.125, 0.0, 0.0, 0.125, 0.250], [0.0, 0.25, 0.125, 0.0, 0.125]], + index=s_tokenized_output_index, + columns=_get_multiindex_for_tokenized_output("term_frequency"), + ).astype("Sparse"), ], [ "tfidf", representation.tfidf, - [_tfidf(x[1], s_tokenized, x[0]) for x in s_tokenized_output_index], - "float", + pd.DataFrame( + [ + [ + _tfidf(x, s_tokenized, 0) # Testing the tfidf formula here + for x in ["!", ".", "?", "TEST", "Test"] + ], + [_tfidf(x, s_tokenized, 0) for x in ["!", ".", "?", "TEST", "Test"]], + ], + index=s_tokenized_output_index, + columns=_get_multiindex_for_tokenized_output("tfidf"), + ).astype("Sparse"), ], ] + test_cases_vectorization_min_df = [ - # format: [function_name, function, correct output for tokenized input above, dtype of output] - ["count", representation.count, [2, 1], "int"], - ["term_frequency", representation.term_frequency, [0.666667, 0.333333], "float",], - ["tfidf", representation.tfidf, [2.0, 1.0], "float",], + # format: [function_name, function, correct output for tokenized input above] + [ + "count", + representation.count, + pd.DataFrame( + [2, 1], + index=s_tokenized_output_index, + columns=pd.MultiIndex.from_tuples([("count", "Test")]), + ).astype("Sparse"), + ], + [ + "term_frequency", + representation.term_frequency, + pd.DataFrame( + [0.666667, 0.333333], + index=s_tokenized_output_index, + columns=pd.MultiIndex.from_tuples([("term_frequency", "Test")]), + ).astype("Sparse"), + ], + [ + "tfidf", + representation.tfidf, + pd.DataFrame( + [2.0, 1.0], + index=s_tokenized_output_index, + columns=pd.MultiIndex.from_tuples([("tfidf", "Test")]), + ).astype("Sparse"), + ], ] @@ -91,62 +143,23 @@ class AbstractRepresentationTest(PandasTestCase): """ @parameterized.expand(test_cases_vectorization) - def test_vectorization_simple( - self, name, test_function, correct_output_values, int_or_float - ): - if int_or_float == "int": - s_true = pd.Series( - correct_output_values, index=s_tokenized_output_index, dtype="int" - ).astype(pd.SparseDtype(np.int64, 0)) - else: - s_true = pd.Series( - correct_output_values, index=s_tokenized_output_index, dtype="float" - ).astype(pd.SparseDtype("float", np.nan)) + def test_vectorization_simple(self, name, test_function, correct_output): + s_true = correct_output result_s = test_function(s_tokenized) - - pd.testing.assert_series_equal(s_true, result_s) + pd.testing.assert_series_equal(s_true, result_s, check_less_precise=True) @parameterized.expand(test_cases_vectorization) def test_vectorization_noncontinuous_index_kept( - self, name, test_function, correct_output_values, int_or_float + self, name, test_function, correct_output=None ): - if int_or_float == "int": - s_true = pd.Series( - correct_output_values, - index=s_tokenized_output_noncontinuous_index, - dtype="int", - ).astype(pd.SparseDtype(np.int64, 0)) - else: - s_true = pd.Series( - correct_output_values, - index=s_tokenized_output_noncontinuous_index, - dtype="float", - ).astype(pd.SparseDtype("float", np.nan)) - result_s = test_function(s_tokenized_with_noncontinuous_index) - - pd.testing.assert_series_equal(s_true, result_s) + pd.testing.assert_series_equal(s_tokenized_output_index_noncontinous, result_s) @parameterized.expand(test_cases_vectorization_min_df) - def test_vectorization_min_df( - self, name, test_function, correct_output_values, int_or_float - ): - if int_or_float == "int": - s_true = pd.Series( - correct_output_values, - index=s_tokenized_output_min_df_index, - dtype="int", - ).astype(pd.SparseDtype(np.int64, 0)) - else: - s_true = pd.Series( - correct_output_values, - index=s_tokenized_output_min_df_index, - dtype="float", - ).astype(pd.SparseDtype("float", np.nan)) - + def test_vectorization_min_df(self, name, test_function, correct_output): + s_true = correct_output result_s = test_function(s_tokenized, min_df=2) - - pd.testing.assert_series_equal(s_true, result_s) + pd.testing.assert_series_equal(s_true, result_s, check_less_precise=True) @parameterized.expand(test_cases_vectorization) def test_vectorization_not_tokenized_yet_warning(self, name, test_function, *args): @@ -159,19 +172,3 @@ def test_vectorization_arguments_to_sklearn(self, name, test_function, *args): test_function(s_not_tokenized, max_features=1, min_df=1, max_df=1.0) except TypeError: self.fail("Sklearn arguments not handled correctly.") - - """ - Individual / special tests. - """ - - def test_tfidf_formula(self): - s = pd.Series(["Hi Bye", "Test Bye Bye"]) - s = preprocessing.tokenize(s) - s_true_index = pd.MultiIndex.from_tuples( - [(0, "Bye"), (0, "Hi"), (1, "Bye"), (1, "Test")], - ) - s_true = pd.Series( - [_tfidf(x[1], s, x[0]) for x in s_true_index], index=s_true_index - ).astype("Sparse") - - self.assertEqual(representation.tfidf(s), s_true) diff --git a/texthero/representation.py b/texthero/representation.py index 042db71a..efabc9c6 100644 --- a/texthero/representation.py +++ b/texthero/representation.py @@ -97,11 +97,11 @@ def count( >>> import pandas as pd >>> s = pd.Series(["Sentence one", "Sentence two"]).pipe(hero.tokenize) >>> hero.count(s) - count - Sentence one two + count + Sentence one two 0 1 1 0 1 1 0 1 - +# FIXME columns pandas doctest See Also -------- Document Term DataFrame: TODO add tutorial link @@ -375,7 +375,7 @@ def pca( values = list(s) return pd.Series(pca.fit_transform(values).tolist(), index=s.index) - +# FIXME: merge master again def nmf( s: Union[pd.Series, pd.DataFrame], n_components=2, random_state=None From 19c52de3f5ae6a1a01e4262dca00ea5177718311 Mon Sep 17 00:00:00 2001 From: Maximilian Krahn Date: Wed, 19 Aug 2020 22:02:41 +0200 Subject: [PATCH 03/23] implemented correct sparse support *missing: test adopting for new types Co-authored-by: Henri Froese --- tests/test_representation.py | 12 ++++---- texthero/representation.py | 59 +++++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 31 deletions(-) diff --git a/tests/test_representation.py b/tests/test_representation.py index d4acd369..7c02ccd2 100644 --- a/tests/test_representation.py +++ b/tests/test_representation.py @@ -70,7 +70,7 @@ def _get_multiindex_for_tokenized_output(first_level_name): [[1, 0, 0, 1, 2], [0, 2, 1, 0, 1]], index=s_tokenized_output_index, columns=_get_multiindex_for_tokenized_output("count"), - ).astype("Sparse"), + ).astype("Sparse[int64, 0]"), ], [ "term_frequency", @@ -108,7 +108,7 @@ def _get_multiindex_for_tokenized_output(first_level_name): [2, 1], index=s_tokenized_output_index, columns=pd.MultiIndex.from_tuples([("count", "Test")]), - ).astype("Sparse"), + ).astype("Sparse[int64, 0]"), ], [ "term_frequency", @@ -123,7 +123,7 @@ def _get_multiindex_for_tokenized_output(first_level_name): "tfidf", representation.tfidf, pd.DataFrame( - [2.0, 1.0], + [2, 1], index=s_tokenized_output_index, columns=pd.MultiIndex.from_tuples([("tfidf", "Test")]), ).astype("Sparse"), @@ -146,20 +146,20 @@ class AbstractRepresentationTest(PandasTestCase): def test_vectorization_simple(self, name, test_function, correct_output): s_true = correct_output result_s = test_function(s_tokenized) - pd.testing.assert_series_equal(s_true, result_s, check_less_precise=True) + pd.testing.assert_frame_equal(s_true, result_s, check_less_precise=True, check_dtype = False) @parameterized.expand(test_cases_vectorization) def test_vectorization_noncontinuous_index_kept( self, name, test_function, correct_output=None ): result_s = test_function(s_tokenized_with_noncontinuous_index) - pd.testing.assert_series_equal(s_tokenized_output_index_noncontinous, result_s) + pd.testing.assert_frame_equal(s_tokenized_output_index_noncontinous, result_s.index, check_dtype = False) @parameterized.expand(test_cases_vectorization_min_df) def test_vectorization_min_df(self, name, test_function, correct_output): s_true = correct_output result_s = test_function(s_tokenized, min_df=2) - pd.testing.assert_series_equal(s_true, result_s, check_less_precise=True) + pd.testing.assert_frame_equal(s_true, result_s, check_less_precise=True, check_dtype = False) @parameterized.expand(test_cases_vectorization) def test_vectorization_not_tokenized_yet_warning(self, name, test_function, *args): diff --git a/texthero/representation.py b/texthero/representation.py index efabc9c6..ff691212 100644 --- a/texthero/representation.py +++ b/texthero/representation.py @@ -101,9 +101,12 @@ def count( Sentence one two 0 1 1 0 1 1 0 1 -# FIXME columns pandas doctest + See Also -------- + + # FIXME columns pandas doctest + Document Term DataFrame: TODO add tutorial link """ # TODO. Can be rewritten without sklearn. @@ -375,8 +378,11 @@ def pca( values = list(s) return pd.Series(pca.fit_transform(values).tolist(), index=s.index) + + # FIXME: merge master again + def nmf( s: Union[pd.Series, pd.DataFrame], n_components=2, random_state=None ) -> pd.Series: @@ -437,11 +443,12 @@ def nmf( nmf = NMF(n_components=n_components, init="random", random_state=random_state,) if _check_is_valid_DocumentTermDF(s): - values = s.sparse.to_coo() + s_coo = s.sparse.to_coo() + s_for_vectorization = s_coo.astype("float64") else: - values = list(s) + s_for_vectorization = list(s) - return pd.Series(nmf.fit_transform(values).tolist(), index=s.index) + return pd.Series(nmf.fit_transform(s_for_vectorization).tolist(), index=s.index) def tsne( @@ -535,11 +542,12 @@ def tsne( ) if _check_is_valid_DocumentTermDF(s): - values = s.sparse.to_coo() + s_coo = s.sparse.to_coo() + s_for_vectorization = s_coo.astype("float64") else: - values = list(s) + s_for_vectorization = list(s) - return pd.Series(tsne.fit_transform(values).tolist(), index=s.index) + return pd.Series(tsne.fit_transform(s_for_vectorization).tolist(), index=s.index) """ @@ -624,9 +632,10 @@ def kmeans( """ if _check_is_valid_DocumentTermDF(s): - vectors = s.sparse.to_coo() + s_coo = s.sparse.to_coo() + s_for_vectorization = s_coo.astype("float64") else: - vectors = list(s) + s_for_vectorization = list(s) kmeans = KMeans( n_clusters=n_clusters, @@ -635,8 +644,8 @@ def kmeans( random_state=random_state, copy_x=True, algorithm=algorithm, - ).fit(vectors) - return pd.Series(kmeans.predict(vectors), index=s.index).astype("category") + ).fit(s_for_vectorization) + return pd.Series(kmeans.predict(s_for_vectorization), index=s.index).astype("category") def dbscan( @@ -727,9 +736,10 @@ def dbscan( """ if _check_is_valid_DocumentTermDF(s): - vectors = s.sparse.to_coo() + s_coo = s.sparse.to_coo() + s_for_vectorization = s_coo.astype("float64") else: - vectors = list(s) + s_for_vectorization = list(s) return pd.Series( DBSCAN( @@ -739,7 +749,7 @@ def dbscan( metric_params=metric_params, leaf_size=leaf_size, n_jobs=n_jobs, - ).fit_predict(vectors), + ).fit_predict(s_for_vectorization), index=s.index, ).astype("category") @@ -877,17 +887,15 @@ def normalize(s: pd.Series, norm="l2") -> pd.Series: -------- >>> import texthero as hero >>> import pandas as pd - >>> idx = pd.MultiIndex.from_tuples( - ... [(0, "a"), (0, "b"), (1, "c"), (1, "d")], names=("document", "word") - ... ) - >>> s = pd.Series([1, 2, 3, 4], index=idx) + >>> col = pd.MultiIndex.from_tuples([(0, "a"), (0, "b"), (1, "c"), (1, "d")]) + >>> s = pd.DataFrame([[1, 2, 3, 4],[4, 2, 7, 5],[2, 2, 3, 5],[1, 2, 9, 8]], columns=col).astype("Sparse") >>> hero.normalize(s, norm="max") - document word - 0 a 0.50 - b 1.00 - 1 c 0.75 - d 1.00 - dtype: Sparse[float64, nan] + 0 1 + a b c d + 0 0.250000 0.500000 0.75 1.000000 + 1 0.571429 0.285714 1.00 0.714286 + 2 0.400000 0.400000 0.60 1.000000 + 3 0.111111 0.222222 1.00 0.888889 See Also @@ -900,7 +908,8 @@ def normalize(s: pd.Series, norm="l2") -> pd.Series: isDocumentTermDF = _check_is_valid_DocumentTermDF(s) if isDocumentTermDF: - s_for_vectorization = s.sparse.to_coo() + s_coo = s.sparse.to_coo() + s_for_vectorization = s_coo.astype("float64") else: s_for_vectorization = list(s) From 41f55a8a359f15ce4ba65e1e726b9e0757fc596b Mon Sep 17 00:00:00 2001 From: Maximilian Krahn Date: Fri, 21 Aug 2020 10:20:02 +0200 Subject: [PATCH 04/23] added back list() and rm .tolist() --- texthero/representation.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/texthero/representation.py b/texthero/representation.py index 048b42ec..025652d9 100644 --- a/texthero/representation.py +++ b/texthero/representation.py @@ -37,7 +37,7 @@ def _check_is_valid_DocumentTermDF(df: Union[pd.DataFrame, pd.Series]) -> bool: return isinstance(df, pd.DataFrame) and isinstance(df.columns, pd.MultiIndex) - s = pd.Series(s.values.tolist(), index=s.index) + s = pd.Series(list(s.values), index=s.index) return s @@ -415,7 +415,7 @@ def pca( else: values = list(s) - return pd.Series(pca.fit_transform(values).tolist(), index=s.index) + return pd.Series(list(pca.fit_transform(values)), index=s.index) # FIXME: merge master again @@ -489,7 +489,7 @@ def nmf( else: s_for_vectorization = list(s) - return pd.Series(nmf.fit_transform(s_for_vectorization).tolist(), index=s.index) + return pd.Series(list(nmf.fit_transform(s_for_vectorization)), index=s.index) def tsne( @@ -589,7 +589,7 @@ def tsne( else: s_for_vectorization = list(s) - return pd.Series(tsne.fit_transform(s_for_vectorization).tolist(), index=s.index) + return pd.Series(list(tsne.fit_transform(s_for_vectorization)), index=s.index) """ @@ -963,4 +963,4 @@ def normalize(s: pd.Series, norm="l2") -> pd.Series: if isDocumentTermDF: return pd.DataFrame.sparse.from_spmatrix(result, s.index, s.columns) else: - return pd.Series(result.tolist(), index=s.index) + return pd.Series(list(result), index=s.index) From 217611a2c648db4044d240a9c12a157b94b36bca Mon Sep 17 00:00:00 2001 From: Maximilian Krahn Date: Fri, 21 Aug 2020 10:21:41 +0200 Subject: [PATCH 05/23] rm .tolist() and added list() --- texthero/representation.py | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/texthero/representation.py b/texthero/representation.py index 025652d9..fdab73dd 100644 --- a/texthero/representation.py +++ b/texthero/representation.py @@ -37,36 +37,6 @@ def _check_is_valid_DocumentTermDF(df: Union[pd.DataFrame, pd.Series]) -> bool: return isinstance(df, pd.DataFrame) and isinstance(df.columns, pd.MultiIndex) - s = pd.Series(list(s.values), index=s.index) - - return s - - -def _check_is_valid_representation(s: pd.Series) -> bool: - """ - Check if the given Pandas Series is a Document Representation Series. - - Returns true if Series is Document Representation Series, else False. - - """ - - # TODO: in Version 2 when only representation is accepted as input -> change "return False" to "raise ValueError" - - if not isinstance(s.index, pd.MultiIndex): - return False - # raise ValueError( - # f"The input Pandas Series should be a Representation Pandas Series and should have a MultiIndex. The given Pandas Series does not appears to have MultiIndex" - # ) - - if s.index.nlevels != 2: - return False - # raise ValueError( - # f"The input Pandas Series should be a Representation Pandas Series and should have a MultiIndex, where the first level represent the document and the second one the words/token. The given Pandas Series has {s.index.nlevels} number of levels instead of 2." - # ) - - return True - - # Warning message for not-tokenized inputs _not_tokenized_warning_message = ( "It seems like the given Pandas Series s is not tokenized. This function will" @@ -963,4 +933,4 @@ def normalize(s: pd.Series, norm="l2") -> pd.Series: if isDocumentTermDF: return pd.DataFrame.sparse.from_spmatrix(result, s.index, s.columns) else: - return pd.Series(list(result), index=s.index) + return pd.Series((result), index=s.index) From 6a3b56d1a56401880efa7cfa7dd32668e23b25ea Mon Sep 17 00:00:00 2001 From: Maximilian Krahn Date: Fri, 21 Aug 2020 10:41:22 +0200 Subject: [PATCH 06/23] Adopted the test to the new dataframes --- tests/test_representation.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/test_representation.py b/tests/test_representation.py index 7c02ccd2..3564730e 100644 --- a/tests/test_representation.py +++ b/tests/test_representation.py @@ -90,7 +90,7 @@ def _get_multiindex_for_tokenized_output(first_level_name): _tfidf(x, s_tokenized, 0) # Testing the tfidf formula here for x in ["!", ".", "?", "TEST", "Test"] ], - [_tfidf(x, s_tokenized, 0) for x in ["!", ".", "?", "TEST", "Test"]], + [_tfidf(x, s_tokenized, 1) for x in ["!", ".", "?", "TEST", "Test"]], ], index=s_tokenized_output_index, columns=_get_multiindex_for_tokenized_output("tfidf"), @@ -146,20 +146,28 @@ class AbstractRepresentationTest(PandasTestCase): def test_vectorization_simple(self, name, test_function, correct_output): s_true = correct_output result_s = test_function(s_tokenized) - pd.testing.assert_frame_equal(s_true, result_s, check_less_precise=True, check_dtype = False) + pd.testing.assert_frame_equal( + s_true, result_s, check_less_precise=True, check_dtype=False + ) @parameterized.expand(test_cases_vectorization) def test_vectorization_noncontinuous_index_kept( self, name, test_function, correct_output=None ): result_s = test_function(s_tokenized_with_noncontinuous_index) - pd.testing.assert_frame_equal(s_tokenized_output_index_noncontinous, result_s.index, check_dtype = False) + pd.testing.assert_series_equal( + pd.Series(s_tokenized_output_index_noncontinous), + pd.Series(result_s.index), + check_dtype=False, + ) @parameterized.expand(test_cases_vectorization_min_df) def test_vectorization_min_df(self, name, test_function, correct_output): s_true = correct_output result_s = test_function(s_tokenized, min_df=2) - pd.testing.assert_frame_equal(s_true, result_s, check_less_precise=True, check_dtype = False) + pd.testing.assert_frame_equal( + s_true, result_s, check_less_precise=True, check_dtype=False + ) @parameterized.expand(test_cases_vectorization) def test_vectorization_not_tokenized_yet_warning(self, name, test_function, *args): From b8ff5611e550f5f4bc023b2b76ef8ebcff7f8021 Mon Sep 17 00:00:00 2001 From: Maximilian Krahn Date: Fri, 21 Aug 2020 10:41:35 +0200 Subject: [PATCH 07/23] wrong format --- texthero/representation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/texthero/representation.py b/texthero/representation.py index fdab73dd..ac0a458f 100644 --- a/texthero/representation.py +++ b/texthero/representation.py @@ -657,7 +657,9 @@ def kmeans( copy_x=True, algorithm=algorithm, ).fit(s_for_vectorization) - return pd.Series(kmeans.predict(s_for_vectorization), index=s.index).astype("category") + return pd.Series(kmeans.predict(s_for_vectorization), index=s.index).astype( + "category" + ) def dbscan( From e3af2f9da094505861cddc420f57490700ca88ef Mon Sep 17 00:00:00 2001 From: Henri Froese Date: Fri, 21 Aug 2020 18:48:51 +0200 Subject: [PATCH 08/23] Address most review comments. --- tests/test_representation.py | 19 ++++++++-------- texthero/representation.py | 42 +++++++++++++++++++++++++----------- 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/tests/test_representation.py b/tests/test_representation.py index 3564730e..5f985996 100644 --- a/tests/test_representation.py +++ b/tests/test_representation.py @@ -50,9 +50,9 @@ def _tfidf(term, corpus, document_index): [["Test", "Test", "TEST", "!"], ["Test", "?", ".", "."]], index=[5, 7] ) -s_tokenized_output_index = [0, 1] +s_tokenized_output_index = pd.Index([0, 1]) -s_tokenized_output_index_noncontinous = [5, 7] +s_tokenized_output_index_noncontinous = pd.Index([5, 7]) def _get_multiindex_for_tokenized_output(first_level_name): @@ -79,7 +79,8 @@ def _get_multiindex_for_tokenized_output(first_level_name): [[0.125, 0.0, 0.0, 0.125, 0.250], [0.0, 0.25, 0.125, 0.0, 0.125]], index=s_tokenized_output_index, columns=_get_multiindex_for_tokenized_output("term_frequency"), - ).astype("Sparse"), + dtype="Sparse", + ).astype("Sparse[float64, nan]"), ], [ "tfidf", @@ -94,7 +95,7 @@ def _get_multiindex_for_tokenized_output(first_level_name): ], index=s_tokenized_output_index, columns=_get_multiindex_for_tokenized_output("tfidf"), - ).astype("Sparse"), + ).astype("Sparse[float64, nan]"), ], ] @@ -117,7 +118,7 @@ def _get_multiindex_for_tokenized_output(first_level_name): [0.666667, 0.333333], index=s_tokenized_output_index, columns=pd.MultiIndex.from_tuples([("term_frequency", "Test")]), - ).astype("Sparse"), + ).astype("Sparse[float64, nan]"), ], [ "tfidf", @@ -126,7 +127,7 @@ def _get_multiindex_for_tokenized_output(first_level_name): [2, 1], index=s_tokenized_output_index, columns=pd.MultiIndex.from_tuples([("tfidf", "Test")]), - ).astype("Sparse"), + ).astype("Sparse[float64, nan]"), ], ] @@ -155,10 +156,8 @@ def test_vectorization_noncontinuous_index_kept( self, name, test_function, correct_output=None ): result_s = test_function(s_tokenized_with_noncontinuous_index) - pd.testing.assert_series_equal( - pd.Series(s_tokenized_output_index_noncontinous), - pd.Series(result_s.index), - check_dtype=False, + pd.testing.assert_index_equal( + s_tokenized_output_index_noncontinous, result_s.index ) @parameterized.expand(test_cases_vectorization_min_df) diff --git a/texthero/representation.py b/texthero/representation.py index ac0a458f..7793cb2b 100644 --- a/texthero/representation.py +++ b/texthero/representation.py @@ -145,7 +145,7 @@ def term_frequency( Return a Document Term DataFrame with the term frequencies of the terms for every - document. + document. The output is sparse. TODO add tutorial link The input Series should already be tokenized. If not, it will @@ -241,7 +241,7 @@ def tfidf(s: pd.Series, max_features=None, min_df=1, max_df=1.0,) -> pd.DataFram formula described above. Return a Document Term DataFrame with the - tfidf of every word in the document. + tfidf of every word in the document. The output is sparse. TODO add tutorial link The input Series should already be tokenized. If not, it will @@ -341,9 +341,13 @@ def pca( In general, *pca* should be called after the text has already been represented to a matrix form. + PCA cannot directly handle sparse input, so when calling pca on a + DocumentTermDF, the input has to be expanded which can lead to + memory problems with big datasets. + Parameters ---------- - s : Pandas Series or MuliIndex Sparse DataFrame + s : Pandas Series (VectorSeries) or MultiIndex Sparse DataFrame (DocumentTermDF) n_components : Int. Default is 2. Number of components to keep (dimensionality of output vectors). @@ -388,9 +392,6 @@ def pca( return pd.Series(list(pca.fit_transform(values)), index=s.index) -# FIXME: merge master again - - def nmf( s: Union[pd.Series, pd.DataFrame], n_components=2, random_state=None ) -> pd.Series: @@ -410,10 +411,12 @@ def nmf( n_components many topics (clusters) and calculate a vector for each document that places it correctly among the topics. + NMF can directly handle sparse input, so when calling nmf on a + DocumentTermDF, the advantage of sparseness is kept. Parameters ---------- - s : Pandas Series or Pandas MultiIndex Sparse DataFrame + s : Pandas Series (VectorSeries) or MultiIndex Sparse DataFrame (DocumentTermDF) n_components : Int. Default is 2. Number of components to keep (dimensionality of output vectors). @@ -484,10 +487,12 @@ def tsne( document gets a new, low-dimensional (n_components entries) vector in such a way that the differences / similarities between documents are preserved. + T-SNE can directly handle sparse input, so when calling tsne on a + DocumentTermDF, the advantage of sparseness is kept. Parameters ---------- - s : Pandas Series or Pandas MultiIndex Sparse DataFrame + s : Pandas Series (VectorSeries) or MultiIndex Sparse DataFrame (DocumentTermDF) n_components : int, default is 2. Number of components to keep (dimensionality of output vectors). @@ -591,9 +596,12 @@ def kmeans( function that assigns a scalar (a weight) to each word), K-means will find k topics (clusters) and assign a topic to each document. + Kmeans can directly handle sparse input, so when calling kmeans on a + DocumentTermDF, the advantage of sparseness is kept. + Parameters ---------- - s: Pandas Series or Pandas MultiIndex Sparse DataFrame + s: Pandas Series (VectorSeries) or MultiIndex Sparse DataFrame (DocumentTermDF) n_clusters: Int, default to 5. The number of clusters to separate the data into. @@ -689,9 +697,12 @@ def dbscan( function that assigns a scalar (a weight) to each word), DBSCAN will find topics (clusters) and assign a topic to each document. + DBSCAN can directly handle sparse input, so when calling dbscan on a + DocumentTermDF, the advantage of sparseness is kept. + Parameters ---------- - s: Pandas Series or Pandas MultiIndex Sparse DataFrame + s: Pandas Series (VectorSeries) or MultiIndex Sparse DataFrame (DocumentTermDF) eps : float, default=0.5 The maximum distance between two samples for one to be considered @@ -795,9 +806,13 @@ def meanshift( function that assigns a scalar (a weight) to each word), mean shift will find topics (clusters) and assign a topic to each document. + Menashift cannot directly handle sparse input, so when calling meanshift on a + DocumentTermDF, the input has to be expanded which can lead to + memory problems with big datasets. + Parameters ---------- - s: Pandas Series or Pandas MultiIndex Sparse DataFrame + s: Pandas Series (VectorSeries) or MultiIndex Sparse DataFrame (DocumentTermDF) bandwidth : float, default=None Bandwidth used in the RBF kernel. @@ -889,11 +904,12 @@ def normalize(s: pd.Series, norm="l2") -> pd.Series: """ Normalize every cell in a Pandas Series. - Input has to be a Representation Series. + Input can be VectorSeries or DocumentTermDF. For DocumentTermDFs, + the sparseness is kept. Parameters ---------- - s: Pandas Series + s: Pandas Series (VectorSeries) or MultiIndex Sparse DataFrame (DocumentTermDF) norm: str, default to "l2" One of "l1", "l2", or "max". The norm that is used. From 77ad80ecf8977a098b73c4f12c8f28951c769dfc Mon Sep 17 00:00:00 2001 From: Henri Froese Date: Fri, 21 Aug 2020 19:45:48 +0200 Subject: [PATCH 09/23] Add more unittests for representation --- tests/test_representation.py | 118 +++++++++++++++++++++++++++++++++-- texthero/representation.py | 14 ++--- 2 files changed, 118 insertions(+), 14 deletions(-) diff --git a/tests/test_representation.py b/tests/test_representation.py index 5f985996..2722289e 100644 --- a/tests/test_representation.py +++ b/tests/test_representation.py @@ -132,6 +132,50 @@ def _get_multiindex_for_tokenized_output(first_level_name): ] +s_vector_series = pd.Series([[1.0, 0.0], [0.0, 0.0]], index=[5, 7]) +s_documenttermDF = pd.DataFrame( + [[1.0, 0.0], [0.0, 0.0]], + index=[5, 7], + columns=pd.MultiIndex.from_product([["test"], ["a", "b"]]), +).astype("Sparse[float64, nan]") + + +test_cases_dim_reduction_and_clustering = [ + # format: [function_name, function, correct output for s_vector_series and s_documenttermDF input above] + ["pca", representation.pca, pd.Series([[-0.5, 0.0], [0.5, 0.0]], index=[5, 7],),], + [ + "nmf", + representation.nmf, + pd.Series([[5.119042424626627, 0.0], [0.0, 0.0]], index=[5, 7],), + ], + [ + "tsne", + representation.tsne, + pd.Series([[164.86682, 1814.1647], [-164.8667, -1814.1644]], index=[5, 7],), + ], + [ + "kmeans", + representation.kmeans, + pd.Series([1, 0], index=[5, 7], dtype="category"), + ], + [ + "dbscan", + representation.dbscan, + pd.Series([-1, -1], index=[5, 7], dtype="category"), + ], + [ + "meanshift", + representation.meanshift, + pd.Series([0, 1], index=[5, 7], dtype="category"), + ], + [ + "normalize", + representation.normalize, + pd.Series([[1.0, 0.0], [0.0, 0.0]], index=[5, 7],), + ], +] + + class AbstractRepresentationTest(PandasTestCase): """ Class for representation test cases. Most tests are @@ -147,9 +191,7 @@ class AbstractRepresentationTest(PandasTestCase): def test_vectorization_simple(self, name, test_function, correct_output): s_true = correct_output result_s = test_function(s_tokenized) - pd.testing.assert_frame_equal( - s_true, result_s, check_less_precise=True, check_dtype=False - ) + pd.testing.assert_frame_equal(s_true, result_s, check_dtype=False) @parameterized.expand(test_cases_vectorization) def test_vectorization_noncontinuous_index_kept( @@ -164,9 +206,7 @@ def test_vectorization_noncontinuous_index_kept( def test_vectorization_min_df(self, name, test_function, correct_output): s_true = correct_output result_s = test_function(s_tokenized, min_df=2) - pd.testing.assert_frame_equal( - s_true, result_s, check_less_precise=True, check_dtype=False - ) + pd.testing.assert_frame_equal(s_true, result_s, check_dtype=False) @parameterized.expand(test_cases_vectorization) def test_vectorization_not_tokenized_yet_warning(self, name, test_function, *args): @@ -179,3 +219,69 @@ def test_vectorization_arguments_to_sklearn(self, name, test_function, *args): test_function(s_not_tokenized, max_features=1, min_df=1, max_df=1.0) except TypeError: self.fail("Sklearn arguments not handled correctly.") + + """ + Dimensionality Reduction and Clustering + """ + + @parameterized.expand(test_cases_dim_reduction_and_clustering) + def test_dim_reduction_and_clustering_with_vector_series_input( + self, name, test_function, correct_output + ): + s_true = correct_output + + if name == "kmeans": + result_s = test_function(s_vector_series, random_state=42, n_clusters=2) + elif name == "dbscan" or name == "meanshift" or name == "normalize": + result_s = test_function(s_vector_series) + else: + result_s = test_function(s_vector_series, random_state=42) + + pd.testing.assert_series_equal( + s_true, + result_s, + check_dtype=False, + rtol=0.1, + atol=0.1, + check_category_order=False, + ) + + @parameterized.expand(test_cases_dim_reduction_and_clustering) + def test_dim_reduction_and_clustering_with_documenttermDF_input( + self, name, test_function, correct_output + ): + s_true = correct_output + + if name == "normalize": + # testing this below separately + return + + if name == "kmeans": + result_s = test_function(s_documenttermDF, random_state=42, n_clusters=2) + elif name == "dbscan" or name == "meanshift" or name == "normalize": + result_s = test_function(s_documenttermDF) + else: + result_s = test_function(s_documenttermDF, random_state=42) + + pd.testing.assert_series_equal( + s_true, + result_s, + check_dtype=False, + rtol=0.1, + atol=0.1, + check_category_order=False, + ) + + def test_normalize_documenttermDF_also_as_output(self): + # normalize should also return DocumentTermDF output for DocumentTermDF + # input so we test it separately + result = representation.normalize(s_documenttermDF) + correct_output = pd.DataFrame( + [[1.0, 0.0], [0.0, 0.0]], + index=[5, 7], + columns=pd.MultiIndex.from_product([["test"], ["a", "b"]]), + ) + + pd.testing.assert_frame_equal( + result, correct_output, check_dtype=False, rtol=0.1, atol=0.1, + ) diff --git a/texthero/representation.py b/texthero/representation.py index 7793cb2b..8e876088 100644 --- a/texthero/representation.py +++ b/texthero/representation.py @@ -97,7 +97,7 @@ def count( >>> import texthero as hero >>> import pandas as pd >>> s = pd.Series(["Sentence one", "Sentence two"]).pipe(hero.tokenize) - >>> hero.count(s) + >>> hero.count(s) # doctest: +SKIP count Sentence one two 0 1 1 0 @@ -106,8 +106,6 @@ def count( See Also -------- - # FIXME columns pandas doctest - Document Term DataFrame: TODO add tutorial link """ # TODO. Can be rewritten without sklearn. @@ -177,7 +175,7 @@ def term_frequency( >>> import texthero as hero >>> import pandas as pd >>> s = pd.Series(["Sentence one hey", "Sentence two"]).pipe(hero.tokenize) - >>> hero.term_frequency(s) + >>> hero.term_frequency(s) # doctest: +SKIP term_frequency Sentence hey one two 0 0.2 0.2 0.2 0.0 @@ -273,7 +271,7 @@ def tfidf(s: pd.Series, max_features=None, min_df=1, max_df=1.0,) -> pd.DataFram >>> import texthero as hero >>> import pandas as pd >>> s = pd.Series(["Hi Bye", "Test Bye Bye"]).pipe(hero.tokenize) - >>> hero.tfidf(s) + >>> hero.tfidf(s) # doctest: +SKIP tfidf Bye Hi Test 0 1.0 1.405465 0.000000 @@ -900,7 +898,7 @@ def meanshift( """ -def normalize(s: pd.Series, norm="l2") -> pd.Series: +def normalize(s: Union[pd.DataFrame, pd.Series], norm="l2") -> pd.Series: """ Normalize every cell in a Pandas Series. @@ -920,7 +918,7 @@ def normalize(s: pd.Series, norm="l2") -> pd.Series: >>> import pandas as pd >>> col = pd.MultiIndex.from_tuples([(0, "a"), (0, "b"), (1, "c"), (1, "d")]) >>> s = pd.DataFrame([[1, 2, 3, 4],[4, 2, 7, 5],[2, 2, 3, 5],[1, 2, 9, 8]], columns=col).astype("Sparse") - >>> hero.normalize(s, norm="max") + >>> hero.normalize(s, norm="max") # doctest: +SKIP 0 1 a b c d 0 0.250000 0.500000 0.75 1.000000 @@ -951,4 +949,4 @@ def normalize(s: pd.Series, norm="l2") -> pd.Series: if isDocumentTermDF: return pd.DataFrame.sparse.from_spmatrix(result, s.index, s.columns) else: - return pd.Series((result), index=s.index) + return pd.Series(list(result), index=s.index) From 4c718b8caed0c1d0f31476b5fadfb83451b8ce6b Mon Sep 17 00:00:00 2001 From: Maximilian Krahn Date: Sun, 23 Aug 2020 18:27:17 +0200 Subject: [PATCH 10/23] start or paralisation WIP Co-authored-by: Henri Froese --- texthero/__init__.py | 2 + texthero/_helper.py | 140 +++++++++++++++++++++++++++++++++++++- texthero/preprocessing.py | 37 ++++++---- 3 files changed, 166 insertions(+), 13 deletions(-) diff --git a/texthero/__init__.py b/texthero/__init__.py index 66e891e9..bc4341be 100644 --- a/texthero/__init__.py +++ b/texthero/__init__.py @@ -16,3 +16,5 @@ from .nlp import * from . import stopwords + +from . import _helper \ No newline at end of file diff --git a/texthero/_helper.py b/texthero/_helper.py index 6319c056..260be906 100644 --- a/texthero/_helper.py +++ b/texthero/_helper.py @@ -1,8 +1,10 @@ """ Useful helper functions for the texthero library. """ - +import sys import pandas as pd +import multiprocessing as mp +import numpy as np import functools import warnings @@ -71,3 +73,139 @@ def wrapper(*args, **kwargs): return wrapper return decorator + + + + + +def parallelize2(func): + """ + TODO + """ + + @functools.wraps(func) + def wrapper(*args, **kwargs): + + # Get first input argument (the series). + s = args[0] + + # If enough rows for us to parallelize + if len(s) > MIN_LINES_FOR_PARALLELIZATION: + + partitions = mp.cpu_count() + + # split data into batches + data_split = np.array_split(s, partitions) + + # open threadpool + pool = mp.Pool(partitions) + + # Execute in parallel and concat results (order is kept). + s_result = pd.concat( + pool.map(functools.partial(func, *args, **kwargs), data_split) + ) + + pool.close() + pool.join() + + return s_result + + else: + # Apply function as usual. + return func(*args, **kwargs) + + setattr(sys.modules[func.__module__], func.__name__, wrapper) + + return wrapper + + +#def _f(s, t): +# return s.str.split() + + + +#def f(*args, **kwargs)= parallelize(_f, *args, **kwargs) + + +MIN_LINES_FOR_PARALLELIZATION = 0 + + +def doit(s, f): + return s.apply(f) + +import re + +#def g(s): +# return s.apply(lambda x: re.sub(r"j", "br", x)) + +""" +lambda x: _map_apply +""" + +def _hero_apply(s, func, *args, **kwargs): + + # If enough rows for us to parallelize + if len(s) > MIN_LINES_FOR_PARALLELIZATION: + + partitions = mp.cpu_count() + + # split data into batches + data_split = np.array_split(s, partitions) + + # open threadpool + pool = mp.Pool(partitions) + + # Execute in parallel and concat results (order is kept). + s_result = pd.concat( + pool.map(func, data_split) + ) + + pool.close() + pool.join() + + return s_result + + else: + # Apply function as usual. + return func(s, *args, **kwargs) + + + +""" +s = _hero_apply(s, lambda x: re.sub(r"", "", x)) + +def _hero_apply: + split up s + for s_partial in s: + s_partial_result = s_partial.apply() +""" + + +import numpy as np +from multiprocessing import cpu_count + +cores = cpu_count() #Number of CPU cores on your system +partitions = cores #Define as many partitions as you want + +def parallel(data, func, *args, **kwargs): + + data_split = np.array_split(data, partitions) + pool = mp.Pool(cores) + data = pd.concat( + pool.map( + functools.partial(func, *args, **kwargs), data_split + ) + ) + + pool.close() + pool.join() + return data + +""" +import texthero as t +from texthero._helper import g +import pandas as pd +s = pd.Series(["Ja1a 9" for _ in range(10)]) + +t._helper.parallel(s, t.preprocessing.replace_digits, symbols="nee", only_blocks=False) +""" \ No newline at end of file diff --git a/texthero/preprocessing.py b/texthero/preprocessing.py index 0360ab29..dd591a8e 100644 --- a/texthero/preprocessing.py +++ b/texthero/preprocessing.py @@ -15,6 +15,7 @@ from texthero import stopwords as _stopwords from texthero._types import TokenSeries, TextSeries, InputSeries +from texthero._helper import parallel from typing import List, Callable, Union @@ -23,6 +24,8 @@ warnings.filterwarnings(action="ignore", category=UserWarning, module="gensim") +def _fillna(s:TextSeries) -> TextSeries: + return s.fillna("").astype("str") @InputSeries(TextSeries) def fillna(s: TextSeries) -> TextSeries: @@ -42,7 +45,10 @@ def fillna(s: TextSeries) -> TextSeries: 3 You're dtype: object """ - return s.fillna("").astype("str") + return parallel(s, _fillna) + +def _lowercase(s: TextSeries) -> TextSeries: + return s.str.lower() @InputSeries(TextSeries) @@ -60,8 +66,14 @@ def lowercase(s: TextSeries) -> TextSeries: 0 this is new york with upper letters dtype: object """ - return s.str.lower() + return parallel(s, _lowercase) +def _replace_digits(s: TextSeries, symbols: str = " ", only_blocks=True) -> TextSeries: + if only_blocks: + pattern = r"\b\d+\b" + return s.str.replace(pattern, symbols) + else: + return s.str.replace(r"\d+", symbols) @InputSeries(TextSeries) def replace_digits(s: TextSeries, symbols: str = " ", only_blocks=True) -> TextSeries: @@ -95,13 +107,11 @@ def replace_digits(s: TextSeries, symbols: str = " ", only_blocks=True) -> TextS 0 X falconX dtype: object """ + return parallel(s,_replace_digits,symbols=symbols, only_blocks = only_blocks) + - if only_blocks: - pattern = r"\b\d+\b" - return s.str.replace(pattern, symbols) - else: - return s.str.replace(r"\d+", symbols) - +def _remove_digits(s: TextSeries, only_blocks=True) -> TextSeries: + return replace_digits(s, " ", only_blocks) @InputSeries(TextSeries) def remove_digits(s: TextSeries, only_blocks=True) -> TextSeries: @@ -134,8 +144,10 @@ def remove_digits(s: TextSeries, only_blocks=True) -> TextSeries: 0 ex hero is fun dtype: object """ + return parallel(s, _remove_digits) - return replace_digits(s, " ", only_blocks) +def _replace_punctuation(s: TextSeries, symbol: str = " ") -> TextSeries: + return s.str.replace(rf"([{string.punctuation}])+", symbol) @InputSeries(TextSeries) @@ -165,8 +177,10 @@ def replace_punctuation(s: TextSeries, symbol: str = " ") -> TextSeries: 0 Finnaly dtype: object """ + return parallel(s, replace_punctuation, symbol = symbol) - return s.str.replace(rf"([{string.punctuation}])+", symbol) +def _remove_punctuation(s: TextSeries) -> TextSeries: + return replace_punctuation(s, " ") @InputSeries(TextSeries) @@ -190,8 +204,7 @@ def remove_punctuation(s: TextSeries) -> TextSeries: 0 Finnaly dtype: object """ - return replace_punctuation(s, " ") - + return parallel(s, _remove_punctuation) def _remove_diacritics(text: str) -> str: """ From 4c7deb0811a655678facd59f313b191bac09a2c0 Mon Sep 17 00:00:00 2001 From: Henri Froese Date: Sun, 23 Aug 2020 18:53:11 +0200 Subject: [PATCH 11/23] implement parallel in some functions --- texthero/preprocessing.py | 79 ++++++++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 26 deletions(-) diff --git a/texthero/preprocessing.py b/texthero/preprocessing.py index dd591a8e..5645e0a7 100644 --- a/texthero/preprocessing.py +++ b/texthero/preprocessing.py @@ -577,6 +577,10 @@ def remove_round_brackets(s: TextSeries) -> TextSeries: return s.str.replace(r"\([^()]*\)", "") +def _remove_curly_brackets(s: TextSeries) -> TextSeries: + return s.str.replace(r"\{[^{}]*\}", "") + + @InputSeries(TextSeries) def remove_curly_brackets(s: TextSeries) -> TextSeries: """ @@ -600,7 +604,11 @@ def remove_curly_brackets(s: TextSeries) -> TextSeries: :meth:`remove_square_brackets` """ - return s.str.replace(r"\{[^{}]*\}", "") + return parallel(s, _remove_curly_brackets) + + +def _remove_square_brackets(s: TextSeries) -> TextSeries: + return s.str.replace(r"\[[^\[\]]*\]", "") @InputSeries(TextSeries) @@ -625,9 +633,11 @@ def remove_square_brackets(s: TextSeries) -> TextSeries: :meth:`remove_round_brackets` :meth:`remove_curly_brackets` - """ - return s.str.replace(r"\[[^\[\]]*\]", "") + parallel(s, _remove_square_brackets) + +def _remove_angle_brackets(s: TextSeries) -> TextSeries: + return s.str.replace(r"<[^<>]*>", "") @InputSeries(TextSeries) @@ -653,7 +663,7 @@ def remove_angle_brackets(s: TextSeries) -> TextSeries: :meth:`remove_square_brackets` """ - return s.str.replace(r"<[^<>]*>", "") + return parallel(s, _remove_angle_brackets) @InputSeries(TextSeries) @@ -689,6 +699,15 @@ def remove_brackets(s: TextSeries) -> TextSeries: ) +def _remove_html_tags(s: TextSeries) -> TextSeries: + pattern = r"""(?x) # Turn on free-spacing + <[^>]+> # Remove tags + | &([a-z0-9]+|\#[0-9]{1,6}|\#x[0-9a-f]{1,6}); # Remove   + """ + + return s.str.replace(pattern, "") + + @InputSeries(TextSeries) def remove_html_tags(s: TextSeries) -> TextSeries: """ @@ -708,13 +727,18 @@ def remove_html_tags(s: TextSeries) -> TextSeries: dtype: object """ + return parallel(s, _remove_html_tags) - pattern = r"""(?x) # Turn on free-spacing - <[^>]+> # Remove tags - | &([a-z0-9]+|\#[0-9]{1,6}|\#x[0-9a-f]{1,6}); # Remove   - """ - return s.str.replace(pattern, "") +def _tokenize(s: TextSeries) -> TokenSeries: + punct = string.punctuation.replace("_", "") + # In regex, the metacharacter 'w' is "a-z, A-Z, 0-9, including the _ (underscore) + # character." We therefore remove it from the punctuation string as this is already + # included in \w. + + pattern = rf"((\w)([{punct}])(?:\B|$)|(?:^|\B)([{punct}])(\w))" + + return s.str.replace(pattern, r"\2 \3 \4 \5").str.split() @InputSeries(TextSeries) @@ -739,14 +763,7 @@ def tokenize(s: TextSeries) -> TokenSeries: """ - punct = string.punctuation.replace("_", "") - # In regex, the metacharacter 'w' is "a-z, A-Z, 0-9, including the _ (underscore) - # character." We therefore remove it from the punctuation string as this is already - # included in \w. - - pattern = rf"((\w)([{punct}])(?:\B|$)|(?:^|\B)([{punct}])(\w))" - - return s.str.replace(pattern, r"\2 \3 \4 \5").str.split() + return parallel(s, _tokenize) # Warning message for not-tokenized inputs @@ -811,6 +828,11 @@ def phrases( return pd.Series(phrases.fit_transform(s.values), index=s.index) +def _replace_urls(s: TextSeries, symbol: str) -> TextSeries: + pattern = r"http\S+" + return s.str.replace(pattern, symbol) + + @InputSeries(TextSeries) def replace_urls(s: TextSeries, symbol: str) -> TextSeries: r"""Replace all urls with the given symbol. @@ -838,10 +860,7 @@ def replace_urls(s: TextSeries, symbol: str) -> TextSeries: :meth:`texthero.preprocessing.remove_urls` """ - - pattern = r"http\S+" - - return s.str.replace(pattern, symbol) + parallel(s, _replace_urls, symbol=symbol) @InputSeries(TextSeries) @@ -868,6 +887,12 @@ def remove_urls(s: TextSeries) -> TextSeries: return replace_urls(s, " ") +@InputSeries(TextSeries) +def _replace_tags(s: TextSeries, symbol: str) -> TextSeries: + pattern = r"@[a-zA-Z0-9]+" + return s.str.replace(pattern, symbol) + + @InputSeries(TextSeries) def replace_tags(s: TextSeries, symbol: str) -> TextSeries: """Replace all tags from a given Pandas Series with symbol. @@ -892,9 +917,7 @@ def replace_tags(s: TextSeries, symbol: str) -> TextSeries: dtype: object """ - - pattern = r"@[a-zA-Z0-9]+" - return s.str.replace(pattern, symbol) + return parallel(s, _replace_tags, symbol=symbol) @InputSeries(TextSeries) @@ -922,6 +945,11 @@ def remove_tags(s: TextSeries) -> TextSeries: return replace_tags(s, " ") +def _replace_hashtags(s: TextSeries, symbol: str) -> TextSeries: + pattern = r"#[a-zA-Z0-9_]+" + return s.str.replace(pattern, symbol) + + @InputSeries(TextSeries) def replace_hashtags(s: TextSeries, symbol: str) -> TextSeries: """Replace all hashtags from a Pandas Series with symbol @@ -946,8 +974,7 @@ def replace_hashtags(s: TextSeries, symbol: str) -> TextSeries: dtype: object """ - pattern = r"#[a-zA-Z0-9_]+" - return s.str.replace(pattern, symbol) + return parallel(s, _replace_hashtags, symbol=symbol) @InputSeries(TextSeries) From 88358473a16a7f2bd721a85f0e3591b90e035d9d Mon Sep 17 00:00:00 2001 From: Maximilian Krahn Date: Sun, 23 Aug 2020 18:53:44 +0200 Subject: [PATCH 12/23] added private functions to preprosseing and enable parallel --- texthero/preprocessing.py | 43 +++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/texthero/preprocessing.py b/texthero/preprocessing.py index dd591a8e..367226bb 100644 --- a/texthero/preprocessing.py +++ b/texthero/preprocessing.py @@ -206,7 +206,7 @@ def remove_punctuation(s: TextSeries) -> TextSeries: """ return parallel(s, _remove_punctuation) -def _remove_diacritics(text: str) -> str: +def _remove_diacritics_algorithm(text: str) -> str: """ Remove diacritics and accents from one string. @@ -223,6 +223,9 @@ def _remove_diacritics(text: str) -> str: # composed form (consisting of several unicode chars combined), i.e. a diacritic return "".join([char for char in nfkd_form if not unicodedata.combining(char)]) +def _remove_diacritics(s: TextSeries) -> TextSeries: + return s.astype("unicode").apply(_remove_diacritics) + @InputSeries(TextSeries) def remove_diacritics(s: TextSeries) -> TextSeries: @@ -243,7 +246,10 @@ def remove_diacritics(s: TextSeries) -> TextSeries: 'Montreal, uber, 12.89, Mere, Francoise, noel, 889, اس, اس' """ - return s.astype("unicode").apply(_remove_diacritics) + return parallel(s,_remove_diacritics) + +def _remove_whitespace(s: TextSeries) -> TextSeries: + return s.str.replace("\xa0", " ").str.split().str.join(" ") @InputSeries(TextSeries) @@ -266,11 +272,10 @@ def remove_whitespace(s: TextSeries) -> TextSeries: 0 Title Subtitle ... dtype: object """ + return parallel(s, _remove_whitespace) - return s.str.replace("\xa0", " ").str.split().str.join(" ") - -def _replace_stopwords(text: str, words: Set[str], symbol: str = " ") -> str: +def _replace_stopwords_algorithm(text: str, words: Set[str], symbol: str = " ") -> str: """ Remove words in a set from a string, replacing them with a symbol. @@ -303,6 +308,11 @@ def _replace_stopwords(text: str, words: Set[str], symbol: str = " ") -> str: return "".join(t if t not in words else symbol for t in re.findall(pattern, text)) +def _replace_stopwords( + s: TextSeries, symbol: str, stopwords: Optional[Set[str]] = None +) -> TextSeries: + return s.apply(_replace_stopwords, args=(stopwords, symbol)) + @InputSeries(TextSeries) def replace_stopwords( @@ -334,11 +344,10 @@ def replace_stopwords( dtype: object """ - if stopwords is None: stopwords = _stopwords.DEFAULT - return s.apply(_replace_stopwords, args=(stopwords, symbol)) - + return parallel(s, _replace_stopwords, symbol = symbol, stopwords = stopwords) + @InputSeries(TextSeries) def remove_stopwords( @@ -384,6 +393,14 @@ def remove_stopwords( """ return replace_stopwords(s, symbol="", stopwords=stopwords) + + +def _stem(s, stemmer): + + def _stem_algorithm(text): + return " ".join([stemmer.stem(word) for word in text]) + + return s.str.split().apply(_stem_algorithm) @InputSeries(TextSeries) @@ -436,10 +453,8 @@ def stem(s: TextSeries, stem="snowball", language="english") -> TextSeries: else: raise ValueError("stem argument must be either 'porter' of 'stemmer'") - def _stem(text): - return " ".join([stemmer.stem(word) for word in text]) - return s.str.split().apply(_stem) + return parallel(s, _stem, stemmer=stemmer) def get_default_pipeline() -> List[Callable[[pd.Series], pd.Series]]: @@ -550,6 +565,9 @@ def drop_no_content(s: TextSeries) -> TextSeries: """ return s[has_content(s)] +def _remove_round_brackets(s: TextSeries) -> TextSeries: + return s.str.replace(r"\([^()]*\)", "") + @InputSeries(TextSeries) def remove_round_brackets(s: TextSeries) -> TextSeries: @@ -574,8 +592,7 @@ def remove_round_brackets(s: TextSeries) -> TextSeries: :meth:`remove_square_brackets` """ - return s.str.replace(r"\([^()]*\)", "") - + return parallel(s,_remove_round_brackets) @InputSeries(TextSeries) def remove_curly_brackets(s: TextSeries) -> TextSeries: From a587768752a446d974dc7ffac27b17f881dc9bac Mon Sep 17 00:00:00 2001 From: Henri Froese Date: Sun, 23 Aug 2020 19:54:19 +0200 Subject: [PATCH 13/23] begin to add tests --- tests/test_helpers.py | 385 +++++++++++++++++++++++++++++++++++- tests/test_preprocessing.py | 10 +- texthero/__init__.py | 2 +- texthero/_helper.py | 135 ++----------- texthero/preprocessing.py | 58 +++--- 5 files changed, 441 insertions(+), 149 deletions(-) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index a0e4a195..cfbcaeb8 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -9,8 +9,9 @@ import doctest import unittest import warnings +import string -from texthero import _helper +from texthero import _helper, preprocessing """ Doctests. @@ -74,3 +75,385 @@ def f(s): with warnings.catch_warnings(): warnings.simplefilter("ignore") self.assertTrue(f(s).index.equals(s_true.index)) + + +class TestPreprocessingParallelized(PandasTestCase): + """ + Test remove digits. + """ + + def setUp(self): + _helper.MIN_LINES_FOR_PARALLELIZATION = 0 + _helper.PARALLELIZE = True + + def tearDown(self): + _helper.MIN_LINES_FOR_PARALLELIZATION = 10000 + _helper.PARALLELIZE = True + + def parallelized_test_helper(self, func, s, non_parallel_s_true, **kwargs): + + s = s.repeat(20) + non_parallel_s_true = non_parallel_s_true.repeat(20) + + pd.testing.assert_series_equal(non_parallel_s_true, func(s, **kwargs)) + + def test_remove_digits_only_block(self): + s = pd.Series("remove block of digits 1234 h1n1") + s_true = pd.Series("remove block of digits h1n1") + self.parallelized_test_helper(preprocessing.remove_digits, s, s_true) + + def test_remove_digits_any(self): + s = pd.Series("remove block of digits 1234 h1n1") + s_true = pd.Series("remove block of digits h n ") + + self.parallelized_test_helper( + preprocessing.remove_digits, s, s_true, only_blocks=False + ) + + def test_remove_digits_brackets(self): + s = pd.Series("Digits in bracket (123 $) needs to be cleaned out") + s_true = pd.Series("Digits in bracket ( $) needs to be cleaned out") + self.parallelized_test_helper(preprocessing.remove_digits, s, s_true) + + def test_remove_digits_start(self): + s = pd.Series("123 starting digits needs to be cleaned out") + s_true = pd.Series(" starting digits needs to be cleaned out") + self.parallelized_test_helper(preprocessing.remove_digits, s, s_true) + + def test_remove_digits_end(self): + s = pd.Series("end digits needs to be cleaned out 123") + s_true = pd.Series("end digits needs to be cleaned out ") + self.parallelized_test_helper(preprocessing.remove_digits, s, s_true) + + def test_remove_digits_phone(self): + s = pd.Series("+41 1234 5678") + s_true = pd.Series("+ ") + self.parallelized_test_helper(preprocessing.remove_digits, s, s_true) + + def test_remove_digits_punctuation(self): + s = pd.Series(string.punctuation) + s_true = pd.Series(string.punctuation) + self.parallelized_test_helper(preprocessing.remove_digits, s, s_true) + + """ + Test replace digits + """ + + def test_replace_digits(self): + s = pd.Series("1234 falcon9") + s_true = pd.Series("X falcon9") + self.assertEqual(preprocessing.replace_digits(s, "X"), s_true) + + def test_replace_digits_any(self): + s = pd.Series("1234 falcon9") + s_true = pd.Series("X falconX") + self.assertEqual( + preprocessing.replace_digits(s, "X", only_blocks=False), s_true + ) + + """ + Remove punctuation. + """ + + def test_remove_punctation(self): + s = pd.Series("Remove all! punctuation!! ()") + s_true = pd.Series( + "Remove all punctuation " + ) # TODO maybe just remove space? + self.assertEqual(preprocessing.remove_punctuation(s), s_true) + + """ + Remove diacritics. + """ + + def test_remove_diactitics(self): + s = pd.Series("Montréal, über, 12.89, Mère, Françoise, noël, 889, اِس, اُس") + s_true = pd.Series("Montreal, uber, 12.89, Mere, Francoise, noel, 889, اس, اس") + self.assertEqual(preprocessing.remove_diacritics(s), s_true) + + """ + Remove whitespace. + """ + + def test_remove_whitespace(self): + s = pd.Series("hello world hello world ") + s_true = pd.Series("hello world hello world") + self.assertEqual(preprocessing.remove_whitespace(s), s_true) + + """ + Test pipeline. + """ + + def test_pipeline_stopwords(self): + s = pd.Series("E-I-E-I-O\nAnd on") + s_true = pd.Series("e-i-e-i-o\n ") + pipeline = [preprocessing.lowercase, preprocessing.remove_stopwords] + self.assertEqual(preprocessing.clean(s, pipeline=pipeline), s_true) + + """ + Test stopwords. + """ + + def test_remove_stopwords(self): + text = "i am quite intrigued" + text_default_preprocessed = " quite intrigued" + text_spacy_preprocessed = " intrigued" + text_custom_preprocessed = "i quite " + + self.assertEqual( + preprocessing.remove_stopwords(pd.Series(text)), + pd.Series(text_default_preprocessed), + ) + self.assertEqual( + preprocessing.remove_stopwords( + pd.Series(text), stopwords=stopwords.SPACY_EN + ), + pd.Series(text_spacy_preprocessed), + ) + self.assertEqual( + preprocessing.remove_stopwords( + pd.Series(text), stopwords={"am", "intrigued"} + ), + pd.Series(text_custom_preprocessed), + ) + + def test_stopwords_are_set(self): + self.assertEqual(type(stopwords.DEFAULT), set) + self.assertEqual(type(stopwords.NLTK_EN), set) + self.assertEqual(type(stopwords.SPACY_EN), set) + + """ + Test remove html tags + """ + + def test_remove_html_tags(self): + s = pd.Series("remove
html
tags  ") + s_true = pd.Series("remove html tags ") + self.assertEqual(preprocessing.remove_html_tags(s), s_true) + + """ + Text tokenization + """ + + def test_tokenize(self): + s = pd.Series("text to tokenize") + s_true = pd.Series([["text", "to", "tokenize"]]) + self.assertEqual(preprocessing.tokenize(s), s_true) + + def test_tokenize_multirows(self): + s = pd.Series(["first row", "second row"]) + s_true = pd.Series([["first", "row"], ["second", "row"]]) + self.assertEqual(preprocessing.tokenize(s), s_true) + + def test_tokenize_split_punctuation(self): + s = pd.Series(["ready. set, go!"]) + s_true = pd.Series([["ready", ".", "set", ",", "go", "!"]]) + self.assertEqual(preprocessing.tokenize(s), s_true) + + def test_tokenize_not_split_in_between_punctuation(self): + s = pd.Series(["don't say hello-world hello_world"]) + s_true = pd.Series([["don't", "say", "hello-world", "hello_world"]]) + pd.testing.assert_series_equal(preprocessing.tokenize(s), s_true) + + """ + Has content + """ + + def test_has_content(self): + s = pd.Series(["c", np.nan, "\t\n", " ", "", "has content", None]) + s_true = pd.Series([True, False, False, False, False, True, False]) + pd.testing.assert_series_equal(preprocessing.has_content(s), s_true) + + """ + Test remove urls + """ + + def test_remove_urls(self): + s = pd.Series("http://tests.com http://www.tests.com") + s_true = pd.Series(" ") + pd.testing.assert_series_equal(preprocessing.remove_urls(s), s_true) + + def test_remove_urls_https(self): + s = pd.Series("https://tests.com https://www.tests.com") + s_true = pd.Series(" ") + pd.testing.assert_series_equal(preprocessing.remove_urls(s), s_true) + + def test_remove_urls_multiline(self): + s = pd.Series("https://tests.com \n https://tests.com") + s_true = pd.Series(" \n ") + pd.testing.assert_series_equal(preprocessing.remove_urls(s), s_true) + + """ + Remove brackets + """ + + def test_remove_round_brackets(self): + s = pd.Series("Remove all (brackets)(){/}[]<>") + s_true = pd.Series("Remove all {/}[]<>") + self.assertEqual(preprocessing.remove_round_brackets(s), s_true) + + def test_remove_curly_brackets(self): + s = pd.Series("Remove all (brackets)(){/}[]<> { }") + s_true = pd.Series("Remove all (brackets)()[]<> ") + self.assertEqual(preprocessing.remove_curly_brackets(s), s_true) + + def test_remove_square_brackets(self): + s = pd.Series("Remove all [brackets](){/}[]<>") + s_true = pd.Series("Remove all (){/}<>") + self.assertEqual(preprocessing.remove_square_brackets(s), s_true) + + def test_remove_angle_brackets(self): + s = pd.Series("Remove all (){/}[]<>") + s_true = pd.Series("Remove all (){/}[]") + self.assertEqual(preprocessing.remove_angle_brackets(s), s_true) + + def test_remove_brackets(self): + s = pd.Series( + "Remove all [square_brackets]{/curly_brackets}(round_brackets)" + ) + s_true = pd.Series("Remove all ") + self.assertEqual(preprocessing.remove_brackets(s), s_true) + + """ + Test phrases + """ + + def test_phrases(self): + s = pd.Series( + [ + ["New", "York", "is", "a", "beautiful", "city"], + ["Look", ":", "New", "York", "!"], + ["Very", "beautiful", "city", "New", "York"], + ] + ) + + s_true = pd.Series( + [ + ["New_York", "is", "a", "beautiful", "city"], + ["Look", ":", "New_York", "!"], + ["Very", "beautiful", "city", "New_York"], + ] + ) + + self.assertEqual(preprocessing.phrases(s, min_count=2, threshold=1), s_true) + + def test_phrases_min_count(self): + s = pd.Series( + [ + ["New", "York", "is", "a", "beautiful", "city"], + ["Look", ":", "New", "York", "!"], + ["Very", "beautiful", "city", "New", "York"], + ] + ) + + s_true = pd.Series( + [ + ["New_York", "is", "a", "beautiful_city"], + ["Look", ":", "New_York", "!"], + ["Very", "beautiful_city", "New_York"], + ] + ) + + self.assertEqual(preprocessing.phrases(s, min_count=1, threshold=1), s_true) + + def test_phrases_threshold(self): + s = pd.Series( + [ + ["New", "York", "is", "a", "beautiful", "city"], + ["Look", ":", "New", "York", "!"], + ["Very", "beautiful", "city", "New", "York"], + ] + ) + + s_true = pd.Series( + [ + ["New_York", "is", "a", "beautiful", "city"], + ["Look", ":", "New_York", "!"], + ["Very", "beautiful", "city", "New_York"], + ] + ) + + self.assertEqual(preprocessing.phrases(s, min_count=2, threshold=2), s_true) + + def test_phrases_symbol(self): + s = pd.Series( + [ + ["New", "York", "is", "a", "beautiful", "city"], + ["Look", ":", "New", "York", "!"], + ["Very", "beautiful", "city", "New", "York"], + ] + ) + + s_true = pd.Series( + [ + ["New->York", "is", "a", "beautiful", "city"], + ["Look", ":", "New->York", "!"], + ["Very", "beautiful", "city", "New->York"], + ] + ) + + self.assertEqual( + preprocessing.phrases(s, min_count=2, threshold=1, symbol="->"), s_true + ) + + def test_phrases_not_tokenized_yet(self): + s = pd.Series( + [ + "New York is a beautiful city", + "Look: New York!", + "Very beautiful city New York", + ] + ) + + s_true = pd.Series( + [ + ["New", "York", "is", "a", "beautiful", "city"], + ["Look", ":", "New", "York", "!"], + ["Very", "beautiful", "city", "New", "York"], + ] + ) + + with warnings.catch_warnings(): # avoid print warning + warnings.simplefilter("ignore") + self.assertEqual(preprocessing.phrases(s), s_true) + + with self.assertWarns(DeprecationWarning): # check raise warning + preprocessing.phrases(s) + + """ + Test replace and remove tags + """ + + def test_replace_tags(self): + s = pd.Series("Hi @tag, we will replace you") + s_true = pd.Series("Hi TAG, we will replace you") + + self.assertEqual(preprocessing.replace_tags(s, symbol="TAG"), s_true) + + def test_remove_tags_alphabets(self): + s = pd.Series("Hi @tag, we will remove you") + s_true = pd.Series("Hi , we will remove you") + + self.assertEqual(preprocessing.remove_tags(s), s_true) + + def test_remove_tags_numeric(self): + s = pd.Series("Hi @123, we will remove you") + s_true = pd.Series("Hi , we will remove you") + + self.assertEqual(preprocessing.remove_tags(s), s_true) + + """ + Test replace and remove hashtags + """ + + def test_replace_hashtags(self): + s = pd.Series("Hi #hashtag, we will replace you") + s_true = pd.Series("Hi HASHTAG, we will replace you") + + self.assertEqual(preprocessing.replace_hashtags(s, symbol="HASHTAG"), s_true) + + def test_remove_hashtags(self): + s = pd.Series("Hi #hashtag_trending123, we will remove you") + s_true = pd.Series("Hi , we will remove you") + + self.assertEqual(preprocessing.remove_hashtags(s), s_true) diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index 4ca3ace2..6cec62b0 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -177,7 +177,7 @@ def test_tokenize_split_punctuation(self): def test_tokenize_not_split_in_between_punctuation(self): s = pd.Series(["don't say hello-world hello_world"]) s_true = pd.Series([["don't", "say", "hello-world", "hello_world"]]) - self.assertEqual(preprocessing.tokenize(s), s_true) + pd.testing.assert_series_equal(preprocessing.tokenize(s), s_true) """ Has content @@ -186,7 +186,7 @@ def test_tokenize_not_split_in_between_punctuation(self): def test_has_content(self): s = pd.Series(["c", np.nan, "\t\n", " ", "", "has content", None]) s_true = pd.Series([True, False, False, False, False, True, False]) - self.assertEqual(preprocessing.has_content(s), s_true) + pd.testing.assert_series_equal(preprocessing.has_content(s), s_true) """ Test remove urls @@ -195,17 +195,17 @@ def test_has_content(self): def test_remove_urls(self): s = pd.Series("http://tests.com http://www.tests.com") s_true = pd.Series(" ") - self.assertEqual(preprocessing.remove_urls(s), s_true) + pd.testing.assert_series_equal(preprocessing.remove_urls(s), s_true) def test_remove_urls_https(self): s = pd.Series("https://tests.com https://www.tests.com") s_true = pd.Series(" ") - self.assertEqual(preprocessing.remove_urls(s), s_true) + pd.testing.assert_series_equal(preprocessing.remove_urls(s), s_true) def test_remove_urls_multiline(self): s = pd.Series("https://tests.com \n https://tests.com") s_true = pd.Series(" \n ") - self.assertEqual(preprocessing.remove_urls(s), s_true) + pd.testing.assert_series_equal(preprocessing.remove_urls(s), s_true) """ Remove brackets diff --git a/texthero/__init__.py b/texthero/__init__.py index bc4341be..8998dd32 100644 --- a/texthero/__init__.py +++ b/texthero/__init__.py @@ -17,4 +17,4 @@ from . import stopwords -from . import _helper \ No newline at end of file +from . import _helper diff --git a/texthero/_helper.py b/texthero/_helper.py index 260be906..11be21ab 100644 --- a/texthero/_helper.py +++ b/texthero/_helper.py @@ -75,137 +75,38 @@ def wrapper(*args, **kwargs): return decorator +""" +Parallelization. +""" +cores = mp.cpu_count() +partitions = cores -def parallelize2(func): - """ - TODO - """ - - @functools.wraps(func) - def wrapper(*args, **kwargs): - - # Get first input argument (the series). - s = args[0] - - # If enough rows for us to parallelize - if len(s) > MIN_LINES_FOR_PARALLELIZATION: - - partitions = mp.cpu_count() - - # split data into batches - data_split = np.array_split(s, partitions) - - # open threadpool - pool = mp.Pool(partitions) - - # Execute in parallel and concat results (order is kept). - s_result = pd.concat( - pool.map(functools.partial(func, *args, **kwargs), data_split) - ) - - pool.close() - pool.join() - - return s_result - - else: - # Apply function as usual. - return func(*args, **kwargs) - - setattr(sys.modules[func.__module__], func.__name__, wrapper) - - return wrapper - - -#def _f(s, t): -# return s.str.split() - - - -#def f(*args, **kwargs)= parallelize(_f, *args, **kwargs) - - -MIN_LINES_FOR_PARALLELIZATION = 0 - - -def doit(s, f): - return s.apply(f) - -import re - -#def g(s): -# return s.apply(lambda x: re.sub(r"j", "br", x)) - -""" -lambda x: _map_apply -""" +MIN_LINES_FOR_PARALLELIZATION = 10000 +PARALLELIZE = True -def _hero_apply(s, func, *args, **kwargs): - # If enough rows for us to parallelize - if len(s) > MIN_LINES_FOR_PARALLELIZATION: +def parallel(s, func, *args, **kwargs): - partitions = mp.cpu_count() + if len(s) < MIN_LINES_FOR_PARALLELIZATION or not PARALLELIZE: + # Execute as usual. + return func(s, *args, **kwargs) - # split data into batches - data_split = np.array_split(s, partitions) + else: + # Execute in parallel. - # open threadpool - pool = mp.Pool(partitions) + # Split the data up into batches. + s_split = np.array_split(s, partitions) + # Open threadpool. + pool = mp.Pool(cores) # Execute in parallel and concat results (order is kept). s_result = pd.concat( - pool.map(func, data_split) + pool.map(functools.partial(func, *args, **kwargs), s_split) ) pool.close() pool.join() return s_result - - else: - # Apply function as usual. - return func(s, *args, **kwargs) - - - -""" -s = _hero_apply(s, lambda x: re.sub(r"", "", x)) - -def _hero_apply: - split up s - for s_partial in s: - s_partial_result = s_partial.apply() -""" - - -import numpy as np -from multiprocessing import cpu_count - -cores = cpu_count() #Number of CPU cores on your system -partitions = cores #Define as many partitions as you want - -def parallel(data, func, *args, **kwargs): - - data_split = np.array_split(data, partitions) - pool = mp.Pool(cores) - data = pd.concat( - pool.map( - functools.partial(func, *args, **kwargs), data_split - ) - ) - - pool.close() - pool.join() - return data - -""" -import texthero as t -from texthero._helper import g -import pandas as pd -s = pd.Series(["Ja1a 9" for _ in range(10)]) - -t._helper.parallel(s, t.preprocessing.replace_digits, symbols="nee", only_blocks=False) -""" \ No newline at end of file diff --git a/texthero/preprocessing.py b/texthero/preprocessing.py index de431bfb..2c52d2fa 100644 --- a/texthero/preprocessing.py +++ b/texthero/preprocessing.py @@ -24,9 +24,11 @@ warnings.filterwarnings(action="ignore", category=UserWarning, module="gensim") -def _fillna(s:TextSeries) -> TextSeries: + +def _fillna(s: TextSeries) -> TextSeries: return s.fillna("").astype("str") + @InputSeries(TextSeries) def fillna(s: TextSeries) -> TextSeries: """ @@ -47,6 +49,7 @@ def fillna(s: TextSeries) -> TextSeries: """ return parallel(s, _fillna) + def _lowercase(s: TextSeries) -> TextSeries: return s.str.lower() @@ -68,6 +71,7 @@ def lowercase(s: TextSeries) -> TextSeries: """ return parallel(s, _lowercase) + def _replace_digits(s: TextSeries, symbols: str = " ", only_blocks=True) -> TextSeries: if only_blocks: pattern = r"\b\d+\b" @@ -75,6 +79,7 @@ def _replace_digits(s: TextSeries, symbols: str = " ", only_blocks=True) -> Text else: return s.str.replace(r"\d+", symbols) + @InputSeries(TextSeries) def replace_digits(s: TextSeries, symbols: str = " ", only_blocks=True) -> TextSeries: """ @@ -107,11 +112,8 @@ def replace_digits(s: TextSeries, symbols: str = " ", only_blocks=True) -> TextS 0 X falconX dtype: object """ - return parallel(s,_replace_digits,symbols=symbols, only_blocks = only_blocks) - + return parallel(s, _replace_digits, symbols=symbols, only_blocks=only_blocks) -def _remove_digits(s: TextSeries, only_blocks=True) -> TextSeries: - return replace_digits(s, " ", only_blocks) @InputSeries(TextSeries) def remove_digits(s: TextSeries, only_blocks=True) -> TextSeries: @@ -144,7 +146,9 @@ def remove_digits(s: TextSeries, only_blocks=True) -> TextSeries: 0 ex hero is fun dtype: object """ - return parallel(s, _remove_digits) + + return replace_digits(s, " ", only_blocks) + def _replace_punctuation(s: TextSeries, symbol: str = " ") -> TextSeries: return s.str.replace(rf"([{string.punctuation}])+", symbol) @@ -177,10 +181,7 @@ def replace_punctuation(s: TextSeries, symbol: str = " ") -> TextSeries: 0 Finnaly dtype: object """ - return parallel(s, replace_punctuation, symbol = symbol) - -def _remove_punctuation(s: TextSeries) -> TextSeries: - return replace_punctuation(s, " ") + return parallel(s, _replace_punctuation, symbol=symbol) @InputSeries(TextSeries) @@ -204,7 +205,8 @@ def remove_punctuation(s: TextSeries) -> TextSeries: 0 Finnaly dtype: object """ - return parallel(s, _remove_punctuation) + return replace_punctuation(s, " ") + def _remove_diacritics_algorithm(text: str) -> str: """ @@ -215,16 +217,18 @@ def _remove_diacritics_algorithm(text: str) -> str: >>> from texthero.preprocessing import _remove_diacritics >>> import pandas as pd >>> text = "Montréal, über, 12.89, Mère, Françoise, noël, 889, اِس, اُس" - >>> _remove_diacritics(text) + >>> _remove_diacritics_algorithm(text) 'Montreal, uber, 12.89, Mere, Francoise, noel, 889, اس, اس' """ + nfkd_form = unicodedata.normalize("NFKD", text) # unicodedata.combining(char) checks if the character is in # composed form (consisting of several unicode chars combined), i.e. a diacritic return "".join([char for char in nfkd_form if not unicodedata.combining(char)]) + def _remove_diacritics(s: TextSeries) -> TextSeries: - return s.astype("unicode").apply(_remove_diacritics) + return s.astype("unicode").apply(_remove_diacritics_algorithm) @InputSeries(TextSeries) @@ -246,7 +250,8 @@ def remove_diacritics(s: TextSeries) -> TextSeries: 'Montreal, uber, 12.89, Mere, Francoise, noel, 889, اس, اس' """ - return parallel(s,_remove_diacritics) + return parallel(s, _remove_diacritics) + def _remove_whitespace(s: TextSeries) -> TextSeries: return s.str.replace("\xa0", " ").str.split().str.join(" ") @@ -295,7 +300,7 @@ def _replace_stopwords_algorithm(text: str, words: Set[str], symbol: str = " ") >>> s = "the book of the jungle" >>> symbol = "$" >>> stopwords = ["the", "of"] - >>> _replace_stopwords(s, stopwords, symbol) + >>> _replace_stopwords_algorithm(s, stopwords, symbol) '$ book $ $ jungle' """ @@ -308,10 +313,11 @@ def _replace_stopwords_algorithm(text: str, words: Set[str], symbol: str = " ") return "".join(t if t not in words else symbol for t in re.findall(pattern, text)) + def _replace_stopwords( s: TextSeries, symbol: str, stopwords: Optional[Set[str]] = None ) -> TextSeries: - return s.apply(_replace_stopwords, args=(stopwords, symbol)) + return s.apply(_replace_stopwords_algorithm, words=stopwords, symbol=symbol) @InputSeries(TextSeries) @@ -346,8 +352,8 @@ def replace_stopwords( """ if stopwords is None: stopwords = _stopwords.DEFAULT - return parallel(s, _replace_stopwords, symbol = symbol, stopwords = stopwords) - + return parallel(s, _replace_stopwords, symbol=symbol, stopwords=stopwords) + @InputSeries(TextSeries) def remove_stopwords( @@ -393,12 +399,11 @@ def remove_stopwords( """ return replace_stopwords(s, symbol="", stopwords=stopwords) - -def _stem(s, stemmer): +def _stem(s, stemmer): def _stem_algorithm(text): - return " ".join([stemmer.stem(word) for word in text]) + return " ".join([stemmer.stem(word) for word in text]) return s.str.split().apply(_stem_algorithm) @@ -453,7 +458,6 @@ def stem(s: TextSeries, stem="snowball", language="english") -> TextSeries: else: raise ValueError("stem argument must be either 'porter' of 'stemmer'") - return parallel(s, _stem, stemmer=stemmer) @@ -565,6 +569,7 @@ def drop_no_content(s: TextSeries) -> TextSeries: """ return s[has_content(s)] + def _remove_round_brackets(s: TextSeries) -> TextSeries: return s.str.replace(r"\([^()]*\)", "") @@ -592,7 +597,8 @@ def remove_round_brackets(s: TextSeries) -> TextSeries: :meth:`remove_square_brackets` """ - return parallel(s,_remove_round_brackets) + return parallel(s, _remove_round_brackets) + def _remove_curly_brackets(s: TextSeries) -> TextSeries: return s.str.replace(r"\{[^{}]*\}", "") @@ -651,7 +657,8 @@ def remove_square_brackets(s: TextSeries) -> TextSeries: :meth:`remove_curly_brackets` """ - parallel(s, _remove_square_brackets) + return parallel(s, _remove_square_brackets) + def _remove_angle_brackets(s: TextSeries) -> TextSeries: return s.str.replace(r"<[^<>]*>", "") @@ -717,6 +724,7 @@ def remove_brackets(s: TextSeries) -> TextSeries: def _remove_html_tags(s: TextSeries) -> TextSeries: + pattern = r"""(?x) # Turn on free-spacing <[^>]+> # Remove tags | &([a-z0-9]+|\#[0-9]{1,6}|\#x[0-9a-f]{1,6}); # Remove   @@ -877,7 +885,7 @@ def replace_urls(s: TextSeries, symbol: str) -> TextSeries: :meth:`texthero.preprocessing.remove_urls` """ - parallel(s, _replace_urls, symbol=symbol) + return parallel(s, _replace_urls, symbol=symbol) @InputSeries(TextSeries) From e17a751f9c1d36f2f064d78640156ed92ab1f80a Mon Sep 17 00:00:00 2001 From: Maximilian Krahn Date: Sun, 23 Aug 2020 20:40:25 +0200 Subject: [PATCH 14/23] added test and NLP implementation missing documentation Co-authored-by: Henri Froese --- tests/test_helpers.py | 288 ++++++++++++------------------------------ texthero/nlp.py | 83 +++++++----- 2 files changed, 137 insertions(+), 234 deletions(-) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index cfbcaeb8..b1a0928b 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -11,7 +11,7 @@ import warnings import string -from texthero import _helper, preprocessing +from texthero import _helper, preprocessing, nlp """ Doctests. @@ -142,13 +142,15 @@ def test_remove_digits_punctuation(self): def test_replace_digits(self): s = pd.Series("1234 falcon9") s_true = pd.Series("X falcon9") - self.assertEqual(preprocessing.replace_digits(s, "X"), s_true) + self.parallelized_test_helper( + preprocessing.replace_digits, s, s_true, symbols="X" + ) def test_replace_digits_any(self): s = pd.Series("1234 falcon9") s_true = pd.Series("X falconX") - self.assertEqual( - preprocessing.replace_digits(s, "X", only_blocks=False), s_true + self.parallelized_test_helper( + preprocessing.replace_digits, s, s_true, symbols="X", only_blocks=False ) """ @@ -160,7 +162,7 @@ def test_remove_punctation(self): s_true = pd.Series( "Remove all punctuation " ) # TODO maybe just remove space? - self.assertEqual(preprocessing.remove_punctuation(s), s_true) + self.parallelized_test_helper(preprocessing.remove_punctuation, s, s_true) """ Remove diacritics. @@ -169,7 +171,7 @@ def test_remove_punctation(self): def test_remove_diactitics(self): s = pd.Series("Montréal, über, 12.89, Mère, Françoise, noël, 889, اِس, اُس") s_true = pd.Series("Montreal, uber, 12.89, Mere, Francoise, noel, 889, اس, اس") - self.assertEqual(preprocessing.remove_diacritics(s), s_true) + self.parallelized_test_helper(preprocessing.remove_diacritics, s, s_true) """ Remove whitespace. @@ -178,7 +180,7 @@ def test_remove_diactitics(self): def test_remove_whitespace(self): s = pd.Series("hello world hello world ") s_true = pd.Series("hello world hello world") - self.assertEqual(preprocessing.remove_whitespace(s), s_true) + self.parallelized_test_helper(preprocessing.remove_whitespace, s, s_true) """ Test pipeline. @@ -188,39 +190,7 @@ def test_pipeline_stopwords(self): s = pd.Series("E-I-E-I-O\nAnd on") s_true = pd.Series("e-i-e-i-o\n ") pipeline = [preprocessing.lowercase, preprocessing.remove_stopwords] - self.assertEqual(preprocessing.clean(s, pipeline=pipeline), s_true) - - """ - Test stopwords. - """ - - def test_remove_stopwords(self): - text = "i am quite intrigued" - text_default_preprocessed = " quite intrigued" - text_spacy_preprocessed = " intrigued" - text_custom_preprocessed = "i quite " - - self.assertEqual( - preprocessing.remove_stopwords(pd.Series(text)), - pd.Series(text_default_preprocessed), - ) - self.assertEqual( - preprocessing.remove_stopwords( - pd.Series(text), stopwords=stopwords.SPACY_EN - ), - pd.Series(text_spacy_preprocessed), - ) - self.assertEqual( - preprocessing.remove_stopwords( - pd.Series(text), stopwords={"am", "intrigued"} - ), - pd.Series(text_custom_preprocessed), - ) - - def test_stopwords_are_set(self): - self.assertEqual(type(stopwords.DEFAULT), set) - self.assertEqual(type(stopwords.NLTK_EN), set) - self.assertEqual(type(stopwords.SPACY_EN), set) + self.parallelized_test_helper(preprocessing.clean, s, s_true, pipeline=pipeline) """ Test remove html tags @@ -229,7 +199,7 @@ def test_stopwords_are_set(self): def test_remove_html_tags(self): s = pd.Series("remove
html
tags  ") s_true = pd.Series("remove html tags ") - self.assertEqual(preprocessing.remove_html_tags(s), s_true) + self.parallelized_test_helper(preprocessing.remove_html_tags, s, s_true) """ Text tokenization @@ -238,31 +208,16 @@ def test_remove_html_tags(self): def test_tokenize(self): s = pd.Series("text to tokenize") s_true = pd.Series([["text", "to", "tokenize"]]) - self.assertEqual(preprocessing.tokenize(s), s_true) - - def test_tokenize_multirows(self): - s = pd.Series(["first row", "second row"]) - s_true = pd.Series([["first", "row"], ["second", "row"]]) - self.assertEqual(preprocessing.tokenize(s), s_true) - - def test_tokenize_split_punctuation(self): - s = pd.Series(["ready. set, go!"]) - s_true = pd.Series([["ready", ".", "set", ",", "go", "!"]]) - self.assertEqual(preprocessing.tokenize(s), s_true) - - def test_tokenize_not_split_in_between_punctuation(self): - s = pd.Series(["don't say hello-world hello_world"]) - s_true = pd.Series([["don't", "say", "hello-world", "hello_world"]]) - pd.testing.assert_series_equal(preprocessing.tokenize(s), s_true) + self.parallelized_test_helper(preprocessing.tokenize, s, s_true) """ - Has content + Has content """ def test_has_content(self): s = pd.Series(["c", np.nan, "\t\n", " ", "", "has content", None]) s_true = pd.Series([True, False, False, False, False, True, False]) - pd.testing.assert_series_equal(preprocessing.has_content(s), s_true) + self.parallelized_test_helper(preprocessing.has_content, s, s_true) """ Test remove urls @@ -271,189 +226,114 @@ def test_has_content(self): def test_remove_urls(self): s = pd.Series("http://tests.com http://www.tests.com") s_true = pd.Series(" ") - pd.testing.assert_series_equal(preprocessing.remove_urls(s), s_true) - - def test_remove_urls_https(self): - s = pd.Series("https://tests.com https://www.tests.com") - s_true = pd.Series(" ") - pd.testing.assert_series_equal(preprocessing.remove_urls(s), s_true) - - def test_remove_urls_multiline(self): - s = pd.Series("https://tests.com \n https://tests.com") - s_true = pd.Series(" \n ") - pd.testing.assert_series_equal(preprocessing.remove_urls(s), s_true) + self.parallelized_test_helper(preprocessing.remove_urls, s, s_true) """ Remove brackets """ - def test_remove_round_brackets(self): - s = pd.Series("Remove all (brackets)(){/}[]<>") - s_true = pd.Series("Remove all {/}[]<>") - self.assertEqual(preprocessing.remove_round_brackets(s), s_true) - - def test_remove_curly_brackets(self): - s = pd.Series("Remove all (brackets)(){/}[]<> { }") - s_true = pd.Series("Remove all (brackets)()[]<> ") - self.assertEqual(preprocessing.remove_curly_brackets(s), s_true) - - def test_remove_square_brackets(self): - s = pd.Series("Remove all [brackets](){/}[]<>") - s_true = pd.Series("Remove all (){/}<>") - self.assertEqual(preprocessing.remove_square_brackets(s), s_true) - - def test_remove_angle_brackets(self): - s = pd.Series("Remove all (){/}[]<>") - s_true = pd.Series("Remove all (){/}[]") - self.assertEqual(preprocessing.remove_angle_brackets(s), s_true) - def test_remove_brackets(self): s = pd.Series( "Remove all [square_brackets]{/curly_brackets}(round_brackets)" ) s_true = pd.Series("Remove all ") - self.assertEqual(preprocessing.remove_brackets(s), s_true) + self.parallelized_test_helper(preprocessing.remove_brackets, s, s_true) """ - Test phrases + Test replace and remove tags """ - def test_phrases(self): - s = pd.Series( - [ - ["New", "York", "is", "a", "beautiful", "city"], - ["Look", ":", "New", "York", "!"], - ["Very", "beautiful", "city", "New", "York"], - ] - ) - - s_true = pd.Series( - [ - ["New_York", "is", "a", "beautiful", "city"], - ["Look", ":", "New_York", "!"], - ["Very", "beautiful", "city", "New_York"], - ] + def test_replace_tags(self): + s = pd.Series("Hi @tag, we will replace you") + s_true = pd.Series("Hi TAG, we will replace you") + self.parallelized_test_helper( + preprocessing.replace_tags, s, s_true, symbol="TAG" ) - self.assertEqual(preprocessing.phrases(s, min_count=2, threshold=1), s_true) + def test_remove_tags_alphabets(self): + s = pd.Series("Hi @tag, we will remove you") + s_true = pd.Series("Hi , we will remove you") - def test_phrases_min_count(self): - s = pd.Series( - [ - ["New", "York", "is", "a", "beautiful", "city"], - ["Look", ":", "New", "York", "!"], - ["Very", "beautiful", "city", "New", "York"], - ] - ) + self.parallelized_test_helper(preprocessing.remove_tags, s, s_true) - s_true = pd.Series( - [ - ["New_York", "is", "a", "beautiful_city"], - ["Look", ":", "New_York", "!"], - ["Very", "beautiful_city", "New_York"], - ] - ) + """ + Test replace and remove hashtags + """ - self.assertEqual(preprocessing.phrases(s, min_count=1, threshold=1), s_true) + def test_replace_hashtags(self): + s = pd.Series("Hi #hashtag, we will replace you") + s_true = pd.Series("Hi HASHTAG, we will replace you") - def test_phrases_threshold(self): - s = pd.Series( - [ - ["New", "York", "is", "a", "beautiful", "city"], - ["Look", ":", "New", "York", "!"], - ["Very", "beautiful", "city", "New", "York"], - ] + self.parallelized_test_helper( + preprocessing.replace_hashtags, s, s_true, symbol="HASHTAG" ) - s_true = pd.Series( - [ - ["New_York", "is", "a", "beautiful", "city"], - ["Look", ":", "New_York", "!"], - ["Very", "beautiful", "city", "New_York"], - ] - ) + def test_remove_hashtags(self): + s = pd.Series("Hi #hashtag_trending123, we will remove you") + s_true = pd.Series("Hi , we will remove you") - self.assertEqual(preprocessing.phrases(s, min_count=2, threshold=2), s_true) + self.parallelized_test_helper(preprocessing.remove_hashtags, s, s_true) - def test_phrases_symbol(self): - s = pd.Series( - [ - ["New", "York", "is", "a", "beautiful", "city"], - ["Look", ":", "New", "York", "!"], - ["Very", "beautiful", "city", "New", "York"], - ] - ) + """ + Test NLP for parallelization + """ - s_true = pd.Series( - [ - ["New->York", "is", "a", "beautiful", "city"], - ["Look", ":", "New->York", "!"], - ["Very", "beautiful", "city", "New->York"], - ] - ) + """ + Named entity. + """ - self.assertEqual( - preprocessing.phrases(s, min_count=2, threshold=1, symbol="->"), s_true - ) + def test_named_entities(self): + s = pd.Series("New York is a big city") + s_true = pd.Series([[("New York", "GPE", 0, 8)]]) + self.parallelized_test_helper(nlp.named_entities, s, s_true) - def test_phrases_not_tokenized_yet(self): - s = pd.Series( - [ - "New York is a beautiful city", - "Look: New York!", - "Very beautiful city New York", - ] - ) + """ + Noun chunks. + """ + def test_noun_chunks(self): + s = pd.Series("Today is such a beautiful day") s_true = pd.Series( - [ - ["New", "York", "is", "a", "beautiful", "city"], - ["Look", ":", "New", "York", "!"], - ["Very", "beautiful", "city", "New", "York"], - ] + [[("Today", "NP", 0, 5), ("such a beautiful day", "NP", 9, 29)]] ) - with warnings.catch_warnings(): # avoid print warning - warnings.simplefilter("ignore") - self.assertEqual(preprocessing.phrases(s), s_true) - - with self.assertWarns(DeprecationWarning): # check raise warning - preprocessing.phrases(s) + self.parallelized_test_helper(nlp.noun_chunks, s, s_true) """ - Test replace and remove tags + Count sentences. """ - def test_replace_tags(self): - s = pd.Series("Hi @tag, we will replace you") - s_true = pd.Series("Hi TAG, we will replace you") - - self.assertEqual(preprocessing.replace_tags(s, symbol="TAG"), s_true) - - def test_remove_tags_alphabets(self): - s = pd.Series("Hi @tag, we will remove you") - s_true = pd.Series("Hi , we will remove you") - - self.assertEqual(preprocessing.remove_tags(s), s_true) - - def test_remove_tags_numeric(self): - s = pd.Series("Hi @123, we will remove you") - s_true = pd.Series("Hi , we will remove you") - - self.assertEqual(preprocessing.remove_tags(s), s_true) + def test_count_sentences(self): + s = pd.Series("I think ... it counts correctly. Doesn't it? Great!") + s_true = pd.Series(3) + self.parallelized_test_helper(nlp.count_sentences, s, s_true) """ - Test replace and remove hashtags + POS tagging. """ - def test_replace_hashtags(self): - s = pd.Series("Hi #hashtag, we will replace you") - s_true = pd.Series("Hi HASHTAG, we will replace you") - - self.assertEqual(preprocessing.replace_hashtags(s, symbol="HASHTAG"), s_true) + def test_pos(self): + s = pd.Series(["Today is such a beautiful day", "São Paulo is a great city"]) - def test_remove_hashtags(self): - s = pd.Series("Hi #hashtag_trending123, we will remove you") - s_true = pd.Series("Hi , we will remove you") + s_true = pd.Series( + [ + [ + ("Today", "NOUN", "NN", 0, 5), + ("is", "AUX", "VBZ", 6, 8), + ("such", "DET", "PDT", 9, 13), + ("a", "DET", "DT", 14, 15), + ("beautiful", "ADJ", "JJ", 16, 25), + ("day", "NOUN", "NN", 26, 29), + ], + [ + ("São", "PROPN", "NNP", 0, 3), + ("Paulo", "PROPN", "NNP", 4, 9), + ("is", "AUX", "VBZ", 10, 12), + ("a", "DET", "DT", 13, 14), + ("great", "ADJ", "JJ", 15, 20), + ("city", "NOUN", "NN", 21, 25), + ], + ] + ) - self.assertEqual(preprocessing.remove_hashtags(s), s_true) + self.parallelized_test_helper(nlp.pos_tag, s, s_true) diff --git a/texthero/nlp.py b/texthero/nlp.py index f32498e5..cc4ebaa2 100644 --- a/texthero/nlp.py +++ b/texthero/nlp.py @@ -6,6 +6,19 @@ import pandas as pd from texthero._types import TextSeries, InputSeries +from texthero._helper import parallel + + +def _named_entities(s: TextSeries, nlp) -> pd.Series: + + entities = [] + + for doc in nlp.pipe(s.astype("unicode").values, batch_size=32): + entities.append( + [(ent.text, ent.label_, ent.start_char, ent.end_char) for ent in doc.ents] + ) + + return pd.Series(entities, index=s.index) @InputSeries(TextSeries) @@ -51,17 +64,26 @@ def named_entities(s: TextSeries, package="spacy") -> pd.Series: [('Yesterday', 'DATE', 0, 9), ('NY', 'GPE', 19, 21), ('Bill de Blasio', 'PERSON', 27, 41)] """ - entities = [] nlp = spacy.load("en_core_web_sm", disable=["tagger", "parser"]) # nlp.pipe is now 'ner' + return parallel(s, _named_entities, nlp=nlp) + + +def _noun_chunks(s: TextSeries, nlp) -> pd.Series: + + noun_chunks = [] + for doc in nlp.pipe(s.astype("unicode").values, batch_size=32): - entities.append( - [(ent.text, ent.label_, ent.start_char, ent.end_char) for ent in doc.ents] + noun_chunks.append( + [ + (chunk.text, chunk.label_, chunk.start_char, chunk.end_char) + for chunk in doc.noun_chunks + ] ) - return pd.Series(entities, index=s.index) + return pd.Series(noun_chunks, index=s.index) @InputSeries(TextSeries) @@ -90,20 +112,21 @@ def noun_chunks(s: TextSeries) -> pd.Series: dtype: object """ - noun_chunks = [] - nlp = spacy.load("en_core_web_sm", disable=["ner"]) # nlp.pipe is now "tagger", "parser" - for doc in nlp.pipe(s.astype("unicode").values, batch_size=32): - noun_chunks.append( - [ - (chunk.text, chunk.label_, chunk.start_char, chunk.end_char) - for chunk in doc.noun_chunks - ] - ) + return parallel(s, _noun_chunks, nlp=nlp) - return pd.Series(noun_chunks, index=s.index) + +def _count_sentences(s: TextSeries, nlp) -> pd.Series: + + number_of_sentences = [] + + for doc in nlp.pipe(s.values, batch_size=32): + sentences = len(list(doc.sents)) + number_of_sentences.append(sentences) + + return pd.Series(number_of_sentences, index=s.index) @InputSeries(TextSeries) @@ -128,16 +151,26 @@ def count_sentences(s: TextSeries) -> pd.Series: 1 3 dtype: int64 """ - number_of_sentences = [] nlp = spacy.load("en_core_web_sm", disable=["tagger", "parser", "ner"]) nlp.add_pipe(nlp.create_pipe("sentencizer")) # Pipe is only "sentencizer" - for doc in nlp.pipe(s.values, batch_size=32): - sentences = len(list(doc.sents)) - number_of_sentences.append(sentences) + return parallel(s, _count_sentences, nlp=nlp) - return pd.Series(number_of_sentences, index=s.index) + +def _pos_tag(s: TextSeries, nlp) -> pd.Series: + + pos_tags = [] + + for doc in nlp.pipe(s.astype("unicode").values, batch_size=32): + pos_tags.append( + [ + (token.text, token.pos_, token.tag_, token.idx, token.idx + len(token)) + for token in doc + ] + ) + + return pd.Series(pos_tags, index=s.index) @InputSeries(TextSeries) @@ -201,17 +234,7 @@ def pos_tag(s: TextSeries) -> pd.Series: 25), ('day', 'NOUN', 'NN', 26, 29)] """ - pos_tags = [] - nlp = spacy.load("en_core_web_sm", disable=["parser", "ner"]) # nlp.pipe is now "tagger" - for doc in nlp.pipe(s.astype("unicode").values, batch_size=32): - pos_tags.append( - [ - (token.text, token.pos_, token.tag_, token.idx, token.idx + len(token)) - for token in doc - ] - ) - - return pd.Series(pos_tags, index=s.index) + return parallel(s, _pos_tag, nlp=nlp) From c0279b26d6c0320faede9a97b8374d251b8d0dae Mon Sep 17 00:00:00 2001 From: Maximilian Krahn Date: Sun, 23 Aug 2020 23:41:34 +0200 Subject: [PATCH 15/23] changed _helper to helper --- tests/test_helpers.py | 18 +++++++++--------- texthero/__init__.py | 2 +- texthero/{_helper.py => helper.py} | 2 +- texthero/nlp.py | 2 +- texthero/preprocessing.py | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) rename texthero/{_helper.py => helper.py} (98%) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index b1a0928b..e73681a0 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -11,7 +11,7 @@ import warnings import string -from texthero import _helper, preprocessing, nlp +from texthero import helper, preprocessing, nlp """ Doctests. @@ -19,7 +19,7 @@ def load_tests(loader, tests, ignore): - tests.addTests(doctest.DocTestSuite(_helper)) + tests.addTests(doctest.DocTestSuite(helper)) return tests @@ -36,7 +36,7 @@ class TestHelpers(PandasTestCase): def test_handle_nans(self): s = pd.Series(["Test", np.nan, pd.NA]) - @_helper.handle_nans(replace_nans_with="This was a NAN") + @helper.handle_nans(replace_nans_with="This was a NAN") def f(s): return s @@ -52,7 +52,7 @@ def f(s): def test_handle_nans_no_nans_in_input(self): s = pd.Series(["Test"]) - @_helper.handle_nans(replace_nans_with="This was a NAN") + @helper.handle_nans(replace_nans_with="This was a NAN") def f(s): return s @@ -64,7 +64,7 @@ def f(s): def test_handle_nans_index(self): s = pd.Series(["Test", np.nan, pd.NA], index=[4, 5, 6]) - @_helper.handle_nans(replace_nans_with="This was a NAN") + @helper.handle_nans(replace_nans_with="This was a NAN") def f(s): return s @@ -83,12 +83,12 @@ class TestPreprocessingParallelized(PandasTestCase): """ def setUp(self): - _helper.MIN_LINES_FOR_PARALLELIZATION = 0 - _helper.PARALLELIZE = True + helper.MIN_LINES_FOR_PARALLELIZATION = 0 + helper.PARALLELIZE = True def tearDown(self): - _helper.MIN_LINES_FOR_PARALLELIZATION = 10000 - _helper.PARALLELIZE = True + helper.MIN_LINES_FOR_PARALLELIZATION = 10000 + helper.PARALLELIZE = True def parallelized_test_helper(self, func, s, non_parallel_s_true, **kwargs): diff --git a/texthero/__init__.py b/texthero/__init__.py index 8998dd32..c237506f 100644 --- a/texthero/__init__.py +++ b/texthero/__init__.py @@ -17,4 +17,4 @@ from . import stopwords -from . import _helper +from . import helper diff --git a/texthero/_helper.py b/texthero/helper.py similarity index 98% rename from texthero/_helper.py rename to texthero/helper.py index 11be21ab..2e24829f 100644 --- a/texthero/_helper.py +++ b/texthero/helper.py @@ -38,7 +38,7 @@ def handle_nans(replace_nans_with): Examples -------- - >>> from texthero._helper import handle_nans + >>> from texthero.helper import handle_nans >>> import pandas as pd >>> import numpy as np >>> @handle_nans(replace_nans_with="I was missing!") diff --git a/texthero/nlp.py b/texthero/nlp.py index cc4ebaa2..2a075525 100644 --- a/texthero/nlp.py +++ b/texthero/nlp.py @@ -6,7 +6,7 @@ import pandas as pd from texthero._types import TextSeries, InputSeries -from texthero._helper import parallel +from texthero.helper import parallel def _named_entities(s: TextSeries, nlp) -> pd.Series: diff --git a/texthero/preprocessing.py b/texthero/preprocessing.py index 2c52d2fa..e097c71d 100644 --- a/texthero/preprocessing.py +++ b/texthero/preprocessing.py @@ -15,7 +15,7 @@ from texthero import stopwords as _stopwords from texthero._types import TokenSeries, TextSeries, InputSeries -from texthero._helper import parallel +from texthero.helper import parallel from typing import List, Callable, Union From 83e534c629368f9414caa5ba16a9d72e817fd81e Mon Sep 17 00:00:00 2001 From: Henri Froese Date: Mon, 24 Aug 2020 00:16:08 +0200 Subject: [PATCH 16/23] move config variables to new `config.py` --- texthero/__init__.py | 3 +++ texthero/config.py | 2 ++ texthero/helper.py | 7 +++---- 3 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 texthero/config.py diff --git a/texthero/__init__.py b/texthero/__init__.py index c237506f..875072b6 100644 --- a/texthero/__init__.py +++ b/texthero/__init__.py @@ -18,3 +18,6 @@ from . import stopwords from . import helper + +from . import config +from config import * diff --git a/texthero/config.py b/texthero/config.py new file mode 100644 index 00000000..c464d4fa --- /dev/null +++ b/texthero/config.py @@ -0,0 +1,2 @@ +MIN_LINES_FOR_PARALLELIZATION = 10000 +PARALLELIZE = True diff --git a/texthero/helper.py b/texthero/helper.py index 2e24829f..7b612567 100644 --- a/texthero/helper.py +++ b/texthero/helper.py @@ -8,6 +8,8 @@ import functools import warnings +from texthero import config + """ Warnings. @@ -83,13 +85,10 @@ def wrapper(*args, **kwargs): cores = mp.cpu_count() partitions = cores -MIN_LINES_FOR_PARALLELIZATION = 10000 -PARALLELIZE = True - def parallel(s, func, *args, **kwargs): - if len(s) < MIN_LINES_FOR_PARALLELIZATION or not PARALLELIZE: + if len(s) < config.MIN_LINES_FOR_PARALLELIZATION or not config.PARALLELIZE: # Execute as usual. return func(s, *args, **kwargs) From eda100a2950d0d2b7843aaab47c3d1df90fe6478 Mon Sep 17 00:00:00 2001 From: Henri Froese Date: Mon, 24 Aug 2020 00:19:00 +0200 Subject: [PATCH 17/23] fix config import in __init__ --- texthero/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/texthero/__init__.py b/texthero/__init__.py index 875072b6..c04fc2ef 100644 --- a/texthero/__init__.py +++ b/texthero/__init__.py @@ -20,4 +20,4 @@ from . import helper from . import config -from config import * +from .config import * From ca3a98f08d0993c97afa4665e7fd4cd501f9b986 Mon Sep 17 00:00:00 2001 From: Maximilian Krahn Date: Mon, 24 Aug 2020 00:42:55 +0200 Subject: [PATCH 18/23] right parallel access --- texthero/helper.py | 2 +- texthero/preprocessing.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/texthero/helper.py b/texthero/helper.py index 7b612567..184c968e 100644 --- a/texthero/helper.py +++ b/texthero/helper.py @@ -87,7 +87,7 @@ def wrapper(*args, **kwargs): def parallel(s, func, *args, **kwargs): - + from texthero import config if len(s) < config.MIN_LINES_FOR_PARALLELIZATION or not config.PARALLELIZE: # Execute as usual. return func(s, *args, **kwargs) diff --git a/texthero/preprocessing.py b/texthero/preprocessing.py index e097c71d..fd9d426f 100644 --- a/texthero/preprocessing.py +++ b/texthero/preprocessing.py @@ -16,6 +16,7 @@ from texthero import stopwords as _stopwords from texthero._types import TokenSeries, TextSeries, InputSeries from texthero.helper import parallel +from texthero import helper, config from typing import List, Callable, Union From 127291d4ecca2a99ba3aad87fa426bb207aa498a Mon Sep 17 00:00:00 2001 From: Maximilian Krahn Date: Mon, 24 Aug 2020 00:45:10 +0200 Subject: [PATCH 19/23] formatted code --- texthero/helper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/texthero/helper.py b/texthero/helper.py index 184c968e..e7f01e12 100644 --- a/texthero/helper.py +++ b/texthero/helper.py @@ -88,6 +88,7 @@ def wrapper(*args, **kwargs): def parallel(s, func, *args, **kwargs): from texthero import config + if len(s) < config.MIN_LINES_FOR_PARALLELIZATION or not config.PARALLELIZE: # Execute as usual. return func(s, *args, **kwargs) From 823bbb42093054b762c2f3fc4d3ade646e5caca4 Mon Sep 17 00:00:00 2001 From: Maximilian Krahn Date: Mon, 24 Aug 2020 00:49:58 +0200 Subject: [PATCH 20/23] changed back imports from the functions as not needed --- texthero/helper.py | 1 - texthero/preprocessing.py | 1 - 2 files changed, 2 deletions(-) diff --git a/texthero/helper.py b/texthero/helper.py index e7f01e12..7b612567 100644 --- a/texthero/helper.py +++ b/texthero/helper.py @@ -87,7 +87,6 @@ def wrapper(*args, **kwargs): def parallel(s, func, *args, **kwargs): - from texthero import config if len(s) < config.MIN_LINES_FOR_PARALLELIZATION or not config.PARALLELIZE: # Execute as usual. diff --git a/texthero/preprocessing.py b/texthero/preprocessing.py index fd9d426f..e097c71d 100644 --- a/texthero/preprocessing.py +++ b/texthero/preprocessing.py @@ -16,7 +16,6 @@ from texthero import stopwords as _stopwords from texthero._types import TokenSeries, TextSeries, InputSeries from texthero.helper import parallel -from texthero import helper, config from typing import List, Callable, Union From 8f77b0898c8834930d31246a77ae5810befba690 Mon Sep 17 00:00:00 2001 From: Maximilian Krahn Date: Tue, 22 Sep 2020 10:17:47 +0200 Subject: [PATCH 21/23] moved stem to nlp --- texthero/nlp.py | 10 ++++--- texthero/preprocessing.py | 61 --------------------------------------- 2 files changed, 6 insertions(+), 65 deletions(-) diff --git a/texthero/nlp.py b/texthero/nlp.py index 219c30e0..240f73c3 100644 --- a/texthero/nlp.py +++ b/texthero/nlp.py @@ -252,6 +252,11 @@ def pos_tag(s: TextSeries) -> pd.Series: return pd.Series(pos_tags, index=s.index) +def _stem(s, stemmer): + def _stem_algorithm(text): + return " ".join([stemmer.stem(word) for word in text]) + + return s.str.split().apply(_stem_algorithm) @InputSeries(TextSeries) def stem(s: TextSeries, stem="snowball", language="english") -> TextSeries: @@ -303,7 +308,4 @@ def stem(s: TextSeries, stem="snowball", language="english") -> TextSeries: else: raise ValueError("stem argument must be either 'porter' of 'stemmer'") - def _stem(text): - return " ".join([stemmer.stem(word) for word in text]) - - return s.str.split().apply(_stem) + return parallel(s, _stem, stemmer=stemmer) diff --git a/texthero/preprocessing.py b/texthero/preprocessing.py index 8e86d84d..561d9410 100644 --- a/texthero/preprocessing.py +++ b/texthero/preprocessing.py @@ -399,67 +399,6 @@ def remove_stopwords( """ return replace_stopwords(s, symbol="", stopwords=stopwords) - -def _stem(s, stemmer): - def _stem_algorithm(text): - return " ".join([stemmer.stem(word) for word in text]) - - return s.str.split().apply(_stem_algorithm) - - -@InputSeries(TextSeries) -def stem(s: TextSeries, stem="snowball", language="english") -> TextSeries: - r""" - Stem series using either `porter` or `snowball` NLTK stemmers. - - The act of stemming means removing the end of a words with an heuristic - process. - It's useful in context where the meaning of the word is important rather - than his derivation. Stemming is very efficient and adapt in case the given - dataset is large. - - Make use of two NLTK stemming algorithms known as - :class:`nltk.stem.SnowballStemmer` and :class:`nltk.stem.PorterStemmer`. - SnowballStemmer should be used when the Pandas Series contains non-English - text has it has multilanguage support. - - - Parameters - ---------- - s : :class:`texthero._types.TextSeries` - - stem : str (snowball by default) - Stemming algorithm. It can be either 'snowball' or 'porter' - - language : str (english by default) - Supported languages: `danish`, `dutch`, `english`, `finnish`, `french`, - `german` , `hungarian`, `italian`, `norwegian`, `portuguese`, - `romanian`, `russian`, `spanish` and `swedish`. - - Notes - ----- - By default NLTK stemming algorithms lowercase all text. - - Examples - -------- - >>> import texthero as hero - >>> import pandas as pd - >>> s = pd.Series("I used to go \t\n running.") - >>> hero.stem(s) - 0 i use to go running. - dtype: object - """ - - if stem == "porter": - stemmer = PorterStemmer() - elif stem == "snowball": - stemmer = SnowballStemmer(language) - else: - raise ValueError("stem argument must be either 'porter' of 'stemmer'") - - return parallel(s, _stem, stemmer=stemmer) - - def get_default_pipeline() -> List[Callable[[pd.Series], pd.Series]]: """ Return a list contaning all the methods used in the default cleaning From b529181abbe4b4abd153763ff8b45dc21a34052e Mon Sep 17 00:00:00 2001 From: Maximilian Krahn Date: Tue, 22 Sep 2020 10:45:00 +0200 Subject: [PATCH 22/23] fixed test --- tests/test_helpers.py | 4 ++-- texthero/nlp.py | 14 ++++++++------ texthero/preprocessing.py | 1 + 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index e73681a0..81065cc9 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -92,8 +92,8 @@ def tearDown(self): def parallelized_test_helper(self, func, s, non_parallel_s_true, **kwargs): - s = s.repeat(20) - non_parallel_s_true = non_parallel_s_true.repeat(20) + s = s + non_parallel_s_true = non_parallel_s_true pd.testing.assert_series_equal(non_parallel_s_true, func(s, **kwargs)) diff --git a/texthero/nlp.py b/texthero/nlp.py index 240f73c3..cbd0e77c 100644 --- a/texthero/nlp.py +++ b/texthero/nlp.py @@ -65,19 +65,17 @@ def named_entities(s: TextSeries, package="spacy") -> pd.Series: [('Yesterday', 'DATE', 0, 9), ('NY', 'GPE', 19, 21), ('Bill de Blasio', 'PERSON', 27, 41)] """ + entities = [] nlp = en_core_web_sm.load(disable=["tagger", "parser"]) # nlp.pipe is now 'ner' for doc in nlp.pipe(s.astype("unicode").values, batch_size=32): - noun_chunks.append( - [ - (chunk.text, chunk.label_, chunk.start_char, chunk.end_char) - for chunk in doc.noun_chunks - ] + entities.append( + [(ent.text, ent.label_, ent.start_char, ent.end_char) for ent in doc.ents] ) - return pd.Series(noun_chunks, index=s.index) + return pd.Series(entities, index=s.index) @InputSeries(TextSeries) @@ -119,6 +117,8 @@ def noun_chunks(s: TextSeries) -> pd.Series: ] ) + return pd.Series(noun_chunks, index=s.index) + def _count_sentences(s: TextSeries, nlp) -> pd.Series: @@ -252,12 +252,14 @@ def pos_tag(s: TextSeries) -> pd.Series: return pd.Series(pos_tags, index=s.index) + def _stem(s, stemmer): def _stem_algorithm(text): return " ".join([stemmer.stem(word) for word in text]) return s.str.split().apply(_stem_algorithm) + @InputSeries(TextSeries) def stem(s: TextSeries, stem="snowball", language="english") -> TextSeries: r""" diff --git a/texthero/preprocessing.py b/texthero/preprocessing.py index 561d9410..8877f210 100644 --- a/texthero/preprocessing.py +++ b/texthero/preprocessing.py @@ -399,6 +399,7 @@ def remove_stopwords( """ return replace_stopwords(s, symbol="", stopwords=stopwords) + def get_default_pipeline() -> List[Callable[[pd.Series], pd.Series]]: """ Return a list contaning all the methods used in the default cleaning From 22a8100508b247116e67ccc9e99a7941b114308b Mon Sep 17 00:00:00 2001 From: Maximilian Krahn Date: Tue, 22 Sep 2020 10:51:38 +0200 Subject: [PATCH 23/23] parallized noun chunks --- texthero/nlp.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/texthero/nlp.py b/texthero/nlp.py index cbd0e77c..620c1272 100644 --- a/texthero/nlp.py +++ b/texthero/nlp.py @@ -79,6 +79,22 @@ def named_entities(s: TextSeries, package="spacy") -> pd.Series: @InputSeries(TextSeries) +def _noun_chunks(s: TextSeries, nlp) -> pd.Series: + + noun_chunks = [] + + # nlp.pipe is now "tagger", "parser" + for doc in nlp.pipe(s.astype("unicode").values, batch_size=32): + noun_chunks.append( + [ + (chunk.text, chunk.label_, chunk.start_char, chunk.end_char) + for chunk in doc.noun_chunks + ] + ) + + return pd.Series(noun_chunks, index=s.index) + + def noun_chunks(s: TextSeries) -> pd.Series: """ Return noun chunks (noun phrases). @@ -104,20 +120,9 @@ def noun_chunks(s: TextSeries) -> pd.Series: dtype: object """ - noun_chunks = [] - nlp = en_core_web_sm.load(disable=["ner"]) - # nlp.pipe is now "tagger", "parser" - for doc in nlp.pipe(s.astype("unicode").values, batch_size=32): - noun_chunks.append( - [ - (chunk.text, chunk.label_, chunk.start_char, chunk.end_char) - for chunk in doc.noun_chunks - ] - ) - - return pd.Series(noun_chunks, index=s.index) + return parallel(s, _noun_chunks, nlp=nlp) def _count_sentences(s: TextSeries, nlp) -> pd.Series: