From ec2dfa8dfc98feabfb9e0758c491caf656a1df4b Mon Sep 17 00:00:00 2001 From: Henri Froese Date: Sun, 12 Jul 2020 22:00:08 +0200 Subject: [PATCH 1/3] Implement Automated Readability Index, Closes #20 New pull request from #46 as we had some Git problems. Input checking done with pd.api.types.is_string_dtype. Not a permanent solution, will be improved by #60 etc. Co-authored-by: Maximilian Krahn --- tests/test_visualization.py | 22 ++++++++++++++++++ texthero/visualization.py | 46 +++++++++++++++++++++++++++++++++++-- 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/tests/test_visualization.py b/tests/test_visualization.py index 4ffa4566..2a16d295 100644 --- a/tests/test_visualization.py +++ b/tests/test_visualization.py @@ -59,3 +59,25 @@ def test_top_words_digits_punctuation(self): def test_wordcloud(self): s = pd.Series("one two three") self.assertEqual(visualization.wordcloud(s), None) + + """ + Test automated readability index + """ + + def test_automated_readability_index(self): + s = pd.Series(["New York is a beautiful city.", "Look: New York!", "Wow"]) + s_true = pd.Series([3.0, 6.0, 0.0]) + self.assertEqual(visualization.automated_readability_index(s), s_true) + + def test_automated_readability_index_index(self): + s = pd.Series( + ["New York is a beautiful city.", "Look: New York!", "Wow"], + index=[5, 6, 7], + ) + self.assertTrue( + visualization.automated_readability_index(s).index.equals(s.index) + ) + + def test_automated_readability_index_numeric(self): + s = pd.Series([1.0, 2.0]) + self.assertRaises(TypeError, visualization.automated_readability_index, s) diff --git a/texthero/visualization.py b/texthero/visualization.py index c363fc17..dbe088fe 100644 --- a/texthero/visualization.py +++ b/texthero/visualization.py @@ -7,7 +7,7 @@ from wordcloud import WordCloud -from texthero import preprocessing +from texthero import preprocessing, nlp import string from matplotlib.colors import LinearSegmentedColormap as lsg @@ -158,7 +158,7 @@ def top_words(s: pd.Series, normalize=False) -> pd.Series: Return a pandas series with index the top words and as value the count. Tokenization: split by space and remove all punctuations that are not between characters. - + Parameters ---------- normalize : @@ -185,3 +185,45 @@ def top_words(s: pd.Series, normalize=False) -> pd.Series: .explode() # one word for each line .value_counts(normalize=normalize) ) + + +def automated_readability_index(s: pd.Series) -> pd.Series: + """ + Calculate the automated readability index (ARI). + + Calculate ARI for each item in the given Pandas Series. Return a Pandas Series with the ARI scores. + Score is NaN if it cannot be computed (e.g. if the number of sentences is 0). + + Examples + -------- + >>> import texthero as hero + >>> import pandas as pd + >>> s = pd.Series(["New York is a beautiful city.", "Look: New York!", "Wow"]) + >>> hero.automated_readability_index(s) + 0 3.0 + 1 6.0 + 2 0.0 + dtype: float64 + + Reference + -------- + `Automated Readability Index `_ + + """ + if not pd.api.types.is_string_dtype(s): + raise TypeError("Non-string values in given Series.") + + words_s = s.str.split().str.len() - 1 + characters_s = s.str.count(r"[a-zA-Z0-9]") # Regex for alphanumeric. + sentences_s = nlp.count_sentences(s) + + score_s = ( + 4.71 * (characters_s / words_s) + 0.5 * (words_s / sentences_s) - 21.43 + ) + score_s = np.ceil(score_s) + + # Pandas does not raise an Error when dividing by zero -> remove + # wrong values by ourselves. + score_s.loc[~np.isfinite(score_s)] = 0 + + return score_s From 28235d998e153d0952c496b1afc927bf7f0fb260 Mon Sep 17 00:00:00 2001 From: Henri Froese Date: Sun, 12 Jul 2020 22:40:53 +0200 Subject: [PATCH 2/3] Improve Automated Readability Index. Fix format --- texthero/visualization.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/texthero/visualization.py b/texthero/visualization.py index dbe088fe..f3a7142e 100644 --- a/texthero/visualization.py +++ b/texthero/visualization.py @@ -3,6 +3,7 @@ """ import pandas as pd +import numpy as np import plotly.express as px from wordcloud import WordCloud @@ -217,9 +218,7 @@ def automated_readability_index(s: pd.Series) -> pd.Series: characters_s = s.str.count(r"[a-zA-Z0-9]") # Regex for alphanumeric. sentences_s = nlp.count_sentences(s) - score_s = ( - 4.71 * (characters_s / words_s) + 0.5 * (words_s / sentences_s) - 21.43 - ) + score_s = 4.71 * (characters_s / words_s) + 0.5 * (words_s / sentences_s) - 21.43 score_s = np.ceil(score_s) # Pandas does not raise an Error when dividing by zero -> remove From c7b632084fc88bf5f2d670c11f2ebe7eda9ef92a Mon Sep 17 00:00:00 2001 From: Henri Froese Date: Mon, 13 Jul 2020 15:54:59 +0200 Subject: [PATCH 3/3] Improve Automated Readability Index Better check for NaNs and non-string values at the beginning. Additional test in test_visualization for NaNs. --- tests/test_visualization.py | 5 +++++ texthero/visualization.py | 8 +++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_visualization.py b/tests/test_visualization.py index 2a16d295..00967d83 100644 --- a/tests/test_visualization.py +++ b/tests/test_visualization.py @@ -1,6 +1,7 @@ import string import pandas as pd +import numpy as np import doctest from texthero import visualization @@ -81,3 +82,7 @@ def test_automated_readability_index_index(self): def test_automated_readability_index_numeric(self): s = pd.Series([1.0, 2.0]) self.assertRaises(TypeError, visualization.automated_readability_index, s) + + def test_automated_readability_index_nan(self): + s = pd.Series(["Test", np.nan]) + self.assertRaises(TypeError, visualization.automated_readability_index, s) diff --git a/texthero/visualization.py b/texthero/visualization.py index f3a7142e..0780ed2b 100644 --- a/texthero/visualization.py +++ b/texthero/visualization.py @@ -193,7 +193,6 @@ def automated_readability_index(s: pd.Series) -> pd.Series: Calculate the automated readability index (ARI). Calculate ARI for each item in the given Pandas Series. Return a Pandas Series with the ARI scores. - Score is NaN if it cannot be computed (e.g. if the number of sentences is 0). Examples -------- @@ -211,8 +210,11 @@ def automated_readability_index(s: pd.Series) -> pd.Series: `Automated Readability Index `_ """ - if not pd.api.types.is_string_dtype(s): - raise TypeError("Non-string values in given Series.") + # Check if type is strings only. + if not s.map(type).eq(str).all(): + raise TypeError( + "Non-string values in series. Use hero.drop_no_content(s) to drop those values." + ) words_s = s.str.split().str.len() - 1 characters_s = s.str.count(r"[a-zA-Z0-9]") # Regex for alphanumeric.