diff --git a/.travis.yml b/.travis.yml index f913f183..c76284b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ jobs: env: PATH=/c/Python38:/c/Python38/Scripts:$PATH install: - pip3 install --upgrade pip # all three OSes agree about 'pip3' - - pip3 install black + - pip3 install black==19.10b0 - pip3 install ".[dev]" . # 'python' points to Python 2.7 on macOS but points to Python 3.8 on Linux and Windows # 'python3' is a 'command not found' error on Windows but 'py' works on Windows only diff --git a/setup.cfg b/setup.cfg index d6103b02..487d2724 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,7 +41,7 @@ install_requires = # TODO pick the correct version. [options.extras_require] dev = - black>=19.10b0 + black==19.10b0 pytest>=4.0.0 Sphinx>=3.0.3 sphinx-markdown-builder>=0.5.4 @@ -49,3 +49,4 @@ dev = nbsphinx parameterized>=0.7.4 coverage + jieba diff --git a/tests/lang/__init__.py b/tests/lang/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/lang/zh/__init__.py b/tests/lang/zh/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/lang/zh/test_indexes.py b/tests/lang/zh/test_indexes.py new file mode 100644 index 00000000..7ac761ed --- /dev/null +++ b/tests/lang/zh/test_indexes.py @@ -0,0 +1,125 @@ +import pandas as pd +from texthero.lang.zh import preprocessing + +from ... import PandasTestCase +import unittest +import string +from parameterized import parameterized + + +# Define valid inputs for different functions. +s_text = pd.Series(["Test"], index=[5]) +s_numeric = pd.Series([5.0], index=[5]) +s_numeric_lists = pd.Series([[5.0, 5.0], [6.0, 6.0]], index=[5, 6]) + +# Define all test cases. Every test case is a list +# of [name of test case, function to test, tuple of valid input for the function]. +# First argument of valid input has to be the Pandas Series where we +# want to keep the index. If this is different for a function, a separate +# test case has to implemented in the class below. +# The tests will be run by AbstractIndexTest below through the @parameterized +# decorator. +# The names will be expanded automatically, so e.g. "named_entities" +# creates test cases test_correct_index_named_entities and test_incorrect_index_named_entities. + + +test_cases_preprocessing = [ + ["fillna", preprocessing.fillna, (s_text,)], + ["remove_whitespace", preprocessing.remove_whitespace, (s_text,)], + ["clean", preprocessing.clean, (s_text,)], + ["remove_html_tags", preprocessing.remove_html_tags, (s_text,)], + ["tokenize", preprocessing.tokenize, (s_text,)], + ["replace_urls", preprocessing.replace_urls, (s_text, "")], + ["remove_urls", preprocessing.remove_urls, (s_text,)], + ["replace_tags", preprocessing.replace_tags, (s_text, "")], + ["remove_tags", preprocessing.remove_tags, (s_text,)], + ["replace_hashtags", preprocessing.replace_hashtags, (s_text, "")], + ["remove_hashtags", preprocessing.remove_hashtags, (s_text,)], +] + +test_cases = test_cases_preprocessing + +# test_cases_nlp = [ +# ["named_entities", nlp.named_entities, (s_text,)], +# ["noun_chunks", nlp.noun_chunks, (s_text,)], +# ] +# +# test_cases_preprocessing = [ +# ["fillna", preprocessing.fillna, (s_text,)], +# ["lowercase", preprocessing.lowercase, (s_text,)], +# ["replace_digits", preprocessing.replace_digits, (s_text, "")], +# ["remove_digits", preprocessing.remove_digits, (s_text,)], +# ["replace_punctuation", preprocessing.replace_punctuation, (s_text, "")], +# ["remove_punctuation", preprocessing.remove_punctuation, (s_text,)], +# ["remove_diacritics", preprocessing.remove_diacritics, (s_text,)], +# ["remove_whitespace", preprocessing.remove_whitespace, (s_text,)], +# ["replace_stopwords", preprocessing.replace_stopwords, (s_text, "")], +# ["remove_stopwords", preprocessing.remove_stopwords, (s_text,)], +# ["stem", preprocessing.stem, (s_text,)], +# ["clean", preprocessing.clean, (s_text,)], +# ["remove_round_brackets", preprocessing.remove_round_brackets, (s_text,)], +# ["remove_curly_brackets", preprocessing.remove_curly_brackets, (s_text,)], +# ["remove_square_brackets", preprocessing.remove_square_brackets, (s_text,)], +# ["remove_angle_brackets", preprocessing.remove_angle_brackets, (s_text,)], +# ["remove_brackets", preprocessing.remove_brackets, (s_text,)], +# ["remove_html_tags", preprocessing.remove_html_tags, (s_text,)], +# ["tokenize", preprocessing.tokenize, (s_text,)], +# ["tokenize_with_phrases", preprocessing.tokenize_with_phrases, (s_text,)], +# ["replace_urls", preprocessing.replace_urls, (s_text, "")], +# ["remove_urls", preprocessing.remove_urls, (s_text,)], +# ["replace_tags", preprocessing.replace_tags, (s_text, "")], +# ["remove_tags", preprocessing.remove_tags, (s_text,)], +# ] +# +# test_cases_representation = [ +# ["count", representation.count, (preprocessing.tokenize(s_text),),], +# [ +# "term_frequency", +# representation.term_frequency, +# (preprocessing.tokenize(s_text),), +# ], +# ["tfidf", representation.tfidf, (preprocessing.tokenize(s_text),)], +# ["pca", representation.pca, (s_numeric_lists, 0)], +# ["nmf", representation.nmf, (s_numeric_lists,)], +# ["tsne", representation.tsne, (s_numeric_lists,)], +# ["kmeans", representation.kmeans, (s_numeric_lists, 1)], +# ["dbscan", representation.dbscan, (s_numeric_lists,)], +# ["meanshift", representation.meanshift, (s_numeric_lists,)], +# ] +# +# test_cases_visualization = [] +# +# test_cases = ( +# test_cases_nlp +# + test_cases_preprocessing +# + test_cases_representation +# + test_cases_visualization +# ) + + +class AbstractIndexTest(PandasTestCase): + """ + Class for index test cases. Tests for all cases + in test_cases whether the input's index is correctly + preserved by the function. Some function's tests + are implemented manually as they take different inputs. + + """ + + """ + Tests defined in test_cases above. + """ + + @parameterized.expand(test_cases) + def test_correct_index(self, name, test_function, valid_input): + s = valid_input[0] + result_s = test_function(*valid_input) + t_same_index = pd.Series(s.values, s.index) + self.assertTrue(result_s.index.equals(t_same_index.index)) + + @parameterized.expand(test_cases) + def test_incorrect_index(self, name, test_function, valid_input): + s = valid_input[0] + result_s = test_function(*valid_input) + t_different_index = pd.Series(s.values, index=None) + self.assertFalse(result_s.index.equals(t_different_index.index)) diff --git a/tests/lang/zh/test_preprocessing.py b/tests/lang/zh/test_preprocessing.py new file mode 100644 index 00000000..50074fcd --- /dev/null +++ b/tests/lang/zh/test_preprocessing.py @@ -0,0 +1,126 @@ +import string + +import pandas as pd +import numpy as np +import doctest + +from texthero.lang.zh import preprocessing, stopwords +from ... import PandasTestCase + + +""" +Test doctest +""" + + +def load_tests(loader, tests, ignore): + tests.addTests(doctest.DocTestSuite(preprocessing)) + return tests + + +class TestPreprocessing(PandasTestCase): + """ + Remove whitespace. + """ + + def test_remove_whitespace(self): + s = pd.Series("早上好啊，\n\t我的朋友。今天我要去吃 KFC。") + s_true = pd.Series("早上好啊， 我的朋友。今天我要去吃 KFC。") + self.assertEqual(preprocessing.remove_whitespace(s), s_true) + + """ + Test pipeline. + """ + + def test_pipeline_stopwords(self): + s = pd.Series("语言是人类区别其他动物的本质特性。\t@中国NLP第一大师\n#如何定义NLP 为什么呢？") + s_true = pd.Series("语言是人类区别其他动物的本质特性。 为什么呢？") + pipeline = [ + preprocessing.remove_whitespace, + preprocessing.remove_hashtags, + preprocessing.remove_tags, + ] + self.assertEqual(preprocessing.clean(s, pipeline=pipeline), s_true) + + """ + Test remove html tags + """ + + def test_remove_html_tags(self): + s = pd.Series(" 中国新闻网
体育
标记  ") + s_true = pd.Series(" 中国新闻网 体育 标记 ") + self.assertEqual(preprocessing.remove_html_tags(s), s_true) + + """ + Text tokenization + """ + + def test_tokenize(self): + s = pd.Series("我昨天吃烤鸭去了。") + s_true = pd.Series([["我", "昨天", "吃", "烤鸭", "去", "了", "。"]]) + self.assertEqual(preprocessing.tokenize(s), s_true) + + def test_tokenize_multirows(self): + s = pd.Series(["今天天气真好", "明天会怎样呢"]) + s_true = pd.Series([["今天天气", "真", "好"], ["明天", "会", "怎样", "呢"]]) + self.assertEqual(preprocessing.tokenize(s), s_true) + + """ + Has content + """ + + def test_has_content(self): + s = pd.Series(["哈哈", np.nan, "\t\n", " ", "", "这有点东西", None]) + s_true = pd.Series([True, False, False, False, False, True, False]) + self.assertEqual(preprocessing.has_content(s), s_true) + + """ + Test remove urls + """ + + def test_remove_urls(self): + s = pd.Series("http://tests.com http://www.tests.com") + s_true = pd.Series(" ") + self.assertEqual(preprocessing.remove_urls(s), s_true) + + def test_remove_urls_https(self): + s = pd.Series("https://tests.com https://www.tests.com") + s_true = pd.Series(" ") + self.assertEqual(preprocessing.remove_urls(s), s_true) + + def test_remove_urls_multiline(self): + s = pd.Series("https://tests.com \n https://tests.com") + s_true = pd.Series(" \n ") + self.assertEqual(preprocessing.remove_urls(s), s_true) + + """ + Test replace and remove tags + """ + + def test_replace_tags(self): + s = pd.Series("你好@马丁123abc佩奇，我要把你取关了。") + s_true = pd.Series("你好TAG，我要把你取关了。") + + self.assertEqual(preprocessing.replace_tags(s, symbol="TAG"), s_true) + + def test_remove_tags(self): + s = pd.Series("你好@马丁123abc佩奇，我要把你取关了。") + s_true = pd.Series("你好 ，我要把你取关了。") + + self.assertEqual(preprocessing.remove_tags(s), s_true) + + """ + Test replace and remove hashtags + """ + + def test_replace_hashtags(self): + s = pd.Series("语言是人类区别其他动物的本质特性。#NLP百科大全") + s_true = pd.Series("语言是人类区别其他动物的本质特性。HASHTAG") + + self.assertEqual(preprocessing.replace_hashtags(s, symbol="HASHTAG"), s_true) + + def test_remove_hashtags(self): + s = pd.Series("语言是人类区别其他动物的本质特性。#NLP百科大全") + s_true = pd.Series("语言是人类区别其他动物的本质特性。 ") + + self.assertEqual(preprocessing.remove_hashtags(s), s_true) diff --git a/texthero/__init__.py b/texthero/__init__.py index 66e891e9..cefa43f7 100644 --- a/texthero/__init__.py +++ b/texthero/__init__.py @@ -16,3 +16,6 @@ from .nlp import * from . import stopwords + +from . import lang +from .lang import * diff --git a/texthero/lang/__init__.py b/texthero/lang/__init__.py new file mode 100644 index 00000000..45467bc7 --- /dev/null +++ b/texthero/lang/__init__.py @@ -0,0 +1 @@ +from . import zh diff --git a/texthero/lang/zh/__init__.py b/texthero/lang/zh/__init__.py new file mode 100644 index 00000000..89e4dfc0 --- /dev/null +++ b/texthero/lang/zh/__init__.py @@ -0,0 +1,18 @@ +"""Texthero: python toolkit for text preprocessing, representation and visualization. + + + +""" +from . import preprocessing +from .preprocessing import * + +# from . import representation +# from .representation import * +# +# from . import visualization +# from .visualization import * +# +# from . import nlp +# from .nlp import * + +from . import stopwords diff --git a/texthero/lang/zh/preprocessing.py b/texthero/lang/zh/preprocessing.py new file mode 100644 index 00000000..1d36fe6d --- /dev/null +++ b/texthero/lang/zh/preprocessing.py @@ -0,0 +1,222 @@ +""" +The texthero.lang.zh.preprocessing module for Chinese. +""" + +import re +import string +from typing import Optional, Set + +import numpy as np +import pandas as pd + +from spacy.lang.zh import Chinese + +import texthero as hero +from texthero._types import TokenSeries, TextSeries, InputSeries + +# Standard functions that supports Chinese +from texthero.preprocessing import ( + fillna, + has_content, + drop_no_content, + remove_whitespace, + remove_html_tags, + replace_urls, + remove_urls, + phrases, +) + +from typing import List, Callable + + +# Ignore gensim annoying warnings +import warnings + +warnings.filterwarnings(action="ignore", category=UserWarning, module="gensim") + + +__all__ = [ + "fillna", + "has_content", + "drop_no_content", + "remove_whitespace", + "remove_html_tags", + "replace_urls", + "remove_urls", + "phrases", + "clean", + "get_default_pipeline", + "remove_hashtags", + "remove_tags", + "replace_hashtags", + "replace_tags", + "tokenize", +] + + +def get_default_pipeline() -> List[Callable[[pd.Series], pd.Series]]: + """ + Return a list with the following functions: + 1. :meth:`texthero.preprocessing.fillna` + 2. :meth:`texthero.preprocessing.remove_whitespace` + 3. :meth:`texthero.preprocessing.tokenize` + + See also + -------- + :meth:`texthero.preprocessing.get_default_pipeline` + """ + return [ + fillna, + remove_whitespace, + tokenize + # remove_stopwords, # TODO: Use a global `remove` function + ] + + +@InputSeries(TextSeries) +def clean(s: TextSeries, pipeline=None) -> TextSeries: + """ + Default pipeline: + 1. :meth:`texthero.preprocessing.fillna` + 2. :meth:`texthero.preprocessing.remove_whitespace` + 3. :meth:`texthero.preprocessing.tokenize` + + Parameters + ---------- + s : :class:`texthero._types.TextSeries` + + pipeline :List[Callable[[Pandas Series], Pandas Series]] + inserting specific pipeline to clean a text + + Examples + -------- + For the default pipeline: + + >>> import texthero.lang.zh as hero + >>> import pandas as pd + >>> s = pd.Series("我昨天吃烤鸭去了。 挺好吃的。") + >>> hero.clean(s) + 0 [我, 昨天, 吃, 烤鸭, 去, 了, 。, 挺好吃, 的, 。] + dtype: object + + See also + -------- + :meth:`texthero.preprocessing.clean` + """ + if not pipeline: + pipeline = get_default_pipeline() + + return hero.preprocessing.clean(s, pipeline) + + +@InputSeries(TextSeries) +def replace_tags(s: TextSeries, symbol: str) -> TextSeries: + """ + Parameters + ---------- + s : :class:`texthero._types.TextSeries` + + symbols : str + Symbols to replace + + Examples + -------- + >>> import texthero.lang.zh as hero + >>> import pandas as pd + >>> s = pd.Series("你好啊@我爱texthero123。") + >>> hero.replace_tags(s, symbol='TAG') + 0 你好啊TAG。 + dtype: object + + See also + -------- + :meth:`texthero.preprocessing.replace_tags` + """ + + pattern = r"@[a-zA-Z0-9\u4e00-\u9fa5]+" + return s.str.replace(pattern, symbol) + + +@InputSeries(TextSeries) +def remove_tags(s: TextSeries) -> TextSeries: + """ + Examples + -------- + >>> import texthero.lang.zh as hero + >>> import pandas as pd + >>> s = pd.Series("你好啊@我爱texthero123。") + >>> hero.remove_tags(s) + 0 你好啊 。 + dtype: object + + See also + -------- + :meth:`texthero.preprocessing.remove_tags` + """ + return replace_tags(s, " ") + + +@InputSeries(TextSeries) +def replace_hashtags(s: TextSeries, symbol: str) -> TextSeries: + """ + Parameters + ---------- + s : :class:`texthero._types.TextSeries` + + symbols : str + Symbols to replace + + Examples + -------- + >>> import texthero.lang.zh as hero + >>> import pandas as pd + >>> s = pd.Series("今天天气真不错#杰克_texthero_123。") + >>> hero.replace_hashtags(s, symbol='HASHTAG') + 0 今天天气真不错HASHTAG。 + dtype: object + + See also + -------- + :meth:`texthero.preprocessing.replace_hashtags` + """ + pattern = r"#[a-zA-Z0-9_\u4e00-\u9fa5]+" + return s.str.replace(pattern, symbol) + + +@InputSeries(TextSeries) +def remove_hashtags(s: TextSeries) -> TextSeries: + """ + Examples + -------- + >>> import texthero.lang.zh as hero + >>> import pandas as pd + >>> s = pd.Series("今天天气真不错#杰克_texthero_123。") + >>> hero.remove_hashtags(s) + 0 今天天气真不错 。 + dtype: object + + See also + -------- + :meth:`texthero.preprocessing.remove_hashtags` + """ + return replace_hashtags(s, " ") + + +@InputSeries(TextSeries) +def tokenize(s: TextSeries) -> TokenSeries: + """ + Examples + -------- + >>> import texthero.lang.zh as hero + >>> import pandas as pd + >>> s = pd.Series(["我昨天吃烤鸭去了。"]) + >>> hero.tokenize(s) + 0 [我, 昨天, 吃, 烤鸭, 去, 了, 。] + dtype: object + + See also + -------- + :meth:`texthero.preprocessing.tokenize` + """ + tokenizer = Chinese() + return s.apply(lambda string: [token.text for token in tokenizer(string)]) diff --git a/texthero/lang/zh/stopwords.py b/texthero/lang/zh/stopwords.py new file mode 100644 index 00000000..ebc3d43e --- /dev/null +++ b/texthero/lang/zh/stopwords.py @@ -0,0 +1,6 @@ +import spacy +from spacy.lang.zh import stop_words as spacy_zh_stopwords + + +SPACY_ZH = spacy_zh_stopwords.STOP_WORDS +DEFAULT = SPACY_ZH