增加Mr, R8, R52, Ohsumed, 20ng数据集的loader ,pipe (#382)

* 增加Mr, R8, R52, Ohsumed, 20ng数据集的loader ,pipe

* "更改_20ng为NG20"

Author: MorningForest

fastNLP/io/loader/__init__.py

@@ -58,6 +58,8 @@
     "ChnSentiCorpLoader",
     "THUCNewsLoader",
     "WeiboSenti100kLoader",
+    "MRLoader",
+    "R8Loader", "R52Loader", "OhsumedLoader", "NG20Loader",
 
     'ConllLoader',
     'Conll2003Loader',
@@ -88,7 +90,8 @@
 ]
 from .classification import CLSBaseLoader, YelpFullLoader, YelpPolarityLoader, AGsNewsLoader, IMDBLoader, \
     SSTLoader, SST2Loader, DBPediaLoader, \
-    ChnSentiCorpLoader, THUCNewsLoader, WeiboSenti100kLoader
+    ChnSentiCorpLoader, THUCNewsLoader, WeiboSenti100kLoader,\
+    MRLoader, R8Loader, R52Loader, OhsumedLoader, NG20Loader
 from .conll import ConllLoader, Conll2003Loader, Conll2003NERLoader, OntoNotesNERLoader, CTBLoader
 from .conll import MsraNERLoader, PeopleDailyNERLoader, WeiboNERLoader
 from .coreference import CoReferenceLoader

fastNLP/io/loader/classification.py

@@ -11,7 +11,13 @@
     "SST2Loader",
     "ChnSentiCorpLoader",
     "THUCNewsLoader",
-    "WeiboSenti100kLoader"
+    "WeiboSenti100kLoader",
+
+    "MRLoader",
+    "R8Loader",
+    "R52Loader",
+    "OhsumedLoader",
+    "NG20Loader",
 ]
 
 
@@ -512,3 +518,123 @@ def download(self) -> str:
         """
         output_dir = self._get_dataset_path('weibo-senti-100k')
         return output_dir
+
+class MRLoader(CLSBaseLoader):
+    def __init__(self):
+        super(MRLoader, self).__init__()
+
+    def download(self, dev_ratio: float = 0.0, re_download: bool = False) -> str:
+        r"""
+            自动下载数据集
+
+            如果dev_ratio不等于0,则根据dev_ratio的值随机将train中的数据取出一部分作为dev数据。
+            下载完成后在output_dir中有train.csv, test.csv, dev.csv三个文件。否则只有train.csv和test.csv
+
+            :param float dev_ratio: 如果路径中没有dev集，从train划分多少作为dev的数据. 如果为0，则不划分dev。
+            :param bool re_download: 是否重新下载数据，以重新切分数据。
+            :return: str, 数据集的目录地址
+        """
+        dataset_name = r'mr'
+        data_dir = self._get_dataset_path(dataset_name=dataset_name)
+        data_dir = _split_dev(dataset_name=dataset_name,
+                              data_dir=data_dir,
+                              dev_ratio=dev_ratio,
+                              re_download=re_download,
+                              suffix='csv')
+        return data_dir
+
+class R8Loader(CLSBaseLoader):
+    def __init__(self):
+        super(R8Loader, self).__init__()
+
+    def download(self, dev_ratio: float = 0.0, re_download: bool = False) -> str:
+        r"""
+            自动下载数据集
+
+            如果dev_ratio不等于0,则根据dev_ratio的值随机将train中的数据取出一部分作为dev数据。
+            下载完成后在output_dir中有train.csv, test.csv, dev.csv三个文件。否则只有train.csv和test.csv
+
+            :param float dev_ratio: 如果路径中没有dev集，从train划分多少作为dev的数据. 如果为0，则不划分dev。
+            :param bool re_download: 是否重新下载数据，以重新切分数据。
+            :return: str, 数据集的目录地址
+        """
+        dataset_name = r'R8'
+        data_dir = self._get_dataset_path(dataset_name=dataset_name)
+        data_dir = _split_dev(dataset_name=dataset_name,
+                              data_dir=data_dir,
+                              dev_ratio=dev_ratio,
+                              re_download=re_download,
+                              suffix='csv')
+        return data_dir
+
+class R52Loader(CLSBaseLoader):
+    def __init__(self):
+        super(R52Loader, self).__init__()
+
+    def download(self, dev_ratio: float = 0.0, re_download: bool = False) -> str:
+        r"""
+            自动下载数据集
+
+            如果dev_ratio不等于0,则根据dev_ratio的值随机将train中的数据取出一部分作为dev数据。
+            下载完成后在output_dir中有train.csv, test.csv, dev.csv三个文件。否则只有train.csv和test.csv
+
+            :param float dev_ratio: 如果路径中没有dev集，从train划分多少作为dev的数据. 如果为0，则不划分dev。
+            :param bool re_download: 是否重新下载数据，以重新切分数据。
+            :return: str, 数据集的目录地址
+        """
+        dataset_name = r'R52'
+        data_dir = self._get_dataset_path(dataset_name=dataset_name)
+        data_dir = _split_dev(dataset_name=dataset_name,
+                              data_dir=data_dir,
+                              dev_ratio=dev_ratio,
+                              re_download=re_download,
+                              suffix='csv')
+        return data_dir
+
+class NG20Loader(CLSBaseLoader):
+    def __init__(self):
+        super(NG20Loader, self).__init__()
+
+    def download(self, dev_ratio: float = 0.0, re_download: bool = False) -> str:
+        r"""
+            自动下载数据集
+
+            如果dev_ratio不等于0,则根据dev_ratio的值随机将train中的数据取出一部分作为dev数据。
+            下载完成后在output_dir中有train.csv, test.csv, dev.csv三个文件。否则只有train.csv和test.csv
+
+            :param float dev_ratio: 如果路径中没有dev集，从train划分多少作为dev的数据. 如果为0，则不划分dev。
+            :param bool re_download: 是否重新下载数据，以重新切分数据。
+            :return: str, 数据集的目录地址
+        """
+        dataset_name = r'20ng'
+        data_dir = self._get_dataset_path(dataset_name=dataset_name)
+        data_dir = _split_dev(dataset_name=dataset_name,
+                              data_dir=data_dir,
+                              dev_ratio=dev_ratio,
+                              re_download=re_download,
+                              suffix='csv')
+        return data_dir
+
+class OhsumedLoader(CLSBaseLoader):
+    def __init__(self):
+        super(OhsumedLoader, self).__init__()
+
+    def download(self, dev_ratio: float = 0.0, re_download: bool = False) -> str:
+        r"""
+            自动下载数据集
+
+            如果dev_ratio不等于0,则根据dev_ratio的值随机将train中的数据取出一部分作为dev数据。
+            下载完成后在output_dir中有train.csv, test.csv, dev.csv三个文件。否则只有train.csv和test.csv
+
+            :param float dev_ratio: 如果路径中没有dev集，从train划分多少作为dev的数据. 如果为0，则不划分dev。
+            :param bool re_download: 是否重新下载数据，以重新切分数据。
+            :return: str, 数据集的目录地址
+        """
+        dataset_name = r'ohsumed'
+        data_dir = self._get_dataset_path(dataset_name=dataset_name)
+        data_dir = _split_dev(dataset_name=dataset_name,
+                              data_dir=data_dir,
+                              dev_ratio=dev_ratio,
+                              re_download=re_download,
+                              suffix='csv')
+        return data_dir

fastNLP/io/pipe/__init__.py

@@ -23,6 +23,7 @@
     "ChnSentiCorpPipe",
     "THUCNewsPipe",
     "WeiboSenti100kPipe",
+    "MRPipe", "R52Pipe", "R8Pipe", "OhsumedPipe", "NG20Loader",
 
     "Conll2003NERPipe",
     "OntoNotesNERPipe",
@@ -59,7 +60,7 @@
 ]
 
 from .classification import CLSBasePipe, YelpFullPipe, YelpPolarityPipe, SSTPipe, SST2Pipe, IMDBPipe, ChnSentiCorpPipe, THUCNewsPipe, \
-    WeiboSenti100kPipe, AGsNewsPipe, DBPediaPipe
+    WeiboSenti100kPipe, AGsNewsPipe, DBPediaPipe, MRPipe, R8Pipe, R52Pipe, OhsumedPipe, NG20Loader
 from .conll import Conll2003NERPipe, OntoNotesNERPipe, MsraNERPipe, WeiboNERPipe, PeopleDailyPipe
 from .conll import Conll2003Pipe
 from .coreference import CoReferencePipe

fastNLP/io/pipe/classification.py

@@ -11,7 +11,8 @@
     'IMDBPipe',
     "ChnSentiCorpPipe",
     "THUCNewsPipe",
-    "WeiboSenti100kPipe"
+    "WeiboSenti100kPipe",
+    "MRPipe", "R8Pipe", "R52Pipe", "OhsumedPipe", "NG20Pipe"
 ]
 
 import re
@@ -28,7 +29,7 @@
 from ..data_bundle import DataBundle
 from ..loader.classification import ChnSentiCorpLoader, THUCNewsLoader, WeiboSenti100kLoader
 from ..loader.classification import IMDBLoader, YelpFullLoader, SSTLoader, SST2Loader, YelpPolarityLoader, \
-    AGsNewsLoader, DBPediaLoader
+    AGsNewsLoader, DBPediaLoader, MRLoader, R52Loader, R8Loader, OhsumedLoader, NG20Loader
 from ...core._logger import logger
 from ...core.const import Const
 from ...core.dataset import DataSet
@@ -827,3 +828,117 @@ def process_from_file(self, paths=None):
         data_bundle = data_loader.load(paths)
         data_bundle = self.process(data_bundle)
         return data_bundle
+
+class MRPipe(CLSBasePipe):
+    def __init__(self, lower: bool = False, tokenizer: str = 'spacy'):
+        r"""
+
+        :param bool lower: 是否将words列的数据小写。
+        :param str tokenizer: 使用什么tokenizer来将句子切分为words. 支持spacy, raw两种。raw即使用空格拆分。
+        """
+        super().__init__(tokenizer=tokenizer, lang='en')
+        self.lower = lower
+
+    def process_from_file(self, paths=None):
+        r"""
+
+        :param paths: 支持路径类型参见 :class:`fastNLP.io.loader.Loader` 的load函数。
+        :return: DataBundle
+        """
+        # 读取数据
+        data_bundle = MRLoader().load(paths)
+        data_bundle = self.process(data_bundle)
+
+        return data_bundle
+
+
+class R8Pipe(CLSBasePipe):
+    def __init__(self, lower: bool = False, tokenizer: str = 'spacy'):
+        r"""
+
+        :param bool lower: 是否将words列的数据小写。
+        :param str tokenizer: 使用什么tokenizer来将句子切分为words. 支持spacy, raw两种。raw即使用空格拆分。
+        """
+        super().__init__(tokenizer=tokenizer, lang='en')
+        self.lower = lower
+
+    def process_from_file(self, paths=None):
+        r"""
+
+        :param paths: 支持路径类型参见 :class:`fastNLP.io.loader.Loader` 的load函数。
+        :return: DataBundle
+        """
+        # 读取数据
+        data_bundle = R8Loader().load(paths)
+        data_bundle = self.process(data_bundle)
+
+        return data_bundle
+
+
+class R52Pipe(CLSBasePipe):
+    def __init__(self, lower: bool = False, tokenizer: str = 'spacy'):
+        r"""
+
+        :param bool lower: 是否将words列的数据小写。
+        :param str tokenizer: 使用什么tokenizer来将句子切分为words. 支持spacy, raw两种。raw即使用空格拆分。
+        """
+        super().__init__(tokenizer=tokenizer, lang='en')
+        self.lower = lower
+
+    def process_from_file(self, paths=None):
+        r"""
+
+        :param paths: 支持路径类型参见 :class:`fastNLP.io.loader.Loader` 的load函数。
+        :return: DataBundle
+        """
+        # 读取数据
+        data_bundle = R52Loader().load(paths)
+        data_bundle = self.process(data_bundle)
+
+        return data_bundle
+
+
+class OhsumedPipe(CLSBasePipe):
+    def __init__(self, lower: bool = False, tokenizer: str = 'spacy'):
+        r"""
+
+        :param bool lower: 是否将words列的数据小写。
+        :param str tokenizer: 使用什么tokenizer来将句子切分为words. 支持spacy, raw两种。raw即使用空格拆分。
+        """
+        super().__init__(tokenizer=tokenizer, lang='en')
+        self.lower = lower
+
+    def process_from_file(self, paths=None):
+        r"""
+
+        :param paths: 支持路径类型参见 :class:`fastNLP.io.loader.Loader` 的load函数。
+        :return: DataBundle
+        """
+        # 读取数据
+        data_bundle = OhsumedLoader().load(paths)
+        data_bundle = self.process(data_bundle)
+
+        return data_bundle
+
+
+class NG20Pipe(CLSBasePipe):
+    def __init__(self, lower: bool = False, tokenizer: str = 'spacy'):
+        r"""
+
+        :param bool lower: 是否将words列的数据小写。
+        :param str tokenizer: 使用什么tokenizer来将句子切分为words. 支持spacy, raw两种。raw即使用空格拆分。
+        """
+        super().__init__(tokenizer=tokenizer, lang='en')
+        self.lower = lower
+
+    def process_from_file(self, paths=None):
+        r"""
+
+        :param paths: 支持路径类型参见 :class:`fastNLP.io.loader.Loader` 的load函数。
+        :return: DataBundle
+        """
+        # 读取数据
+        data_bundle = NG20Loader().load(paths)
+        data_bundle = self.process(data_bundle)
+
+        return data_bundle

tests/data_for_tests/io/20ng/dev.csv

@@ -0,0 +1,6 @@
+talk.religion.misc,"sandvik newton apple com \( kent sandvik \) subject clarification organization cookamunga tourist bureau lines 14 sorry , san jose based rosicrucian order called r c , n't remember time stand r c ordo rosae crucis , words latin order rose cross sigh , seems loosing long term memory otherwise headquarters san jose pretty decent metaphysical bookstore , interested books son loves run around egyptian museum cheers , kent sandvik newton apple com alink ksand private activities net"
+talk.religion.misc,"subject catholic lit nunnally acs harding edu \( john nunnally \) distribution world organization harding university , , ar nntp posting host acs harding edu x news reader vms news 1 reply dlphknob camelot bradley edu 's message 16 apr 93 18 57 20 gmtlines 45 lines 45 dlphknob camelot dlphknob camelot bradley edu writes 1993apr14 476 mtechca maintech com foster mtechca maintech com writes surprised saddened would expect kind behavior evangelical born gospel thumping face 're true christian protestants , always thought catholics behaved better please stoop level e b g f w c protestants , think best way witness strident , intrusive , loud , insulting self righteous \( pleading mode \) please ! i'm begging ! quit confusing religious groups , stop making generalizations ! i'm protestant ! i'm evangelical ! n't believe way way ! i'm creation scientist ! n't think homosexuals hung ! want discuss bible thumpers , would better singling \( making generalizations \) fundamentalists compared actions methodists southern baptists , would think different religions ! sarcasm sure pick correct groups bible thumpers , fundamentalists , southern baptists deserve hasty generalizations prejudicial statements n't pick methodists ! sarcasm please , prejudice thinking people group , please n't write protestants evangelicals ! \( pleading mode \) god wish could get ahold thomas stories n , n tha gb , gb n yvan sasha david cole iv chief research dlphknob camelot bradley edu"
+talk.religion.misc,"sandvik newton apple com \( kent sandvik \) subject alt sex stories literary critical analysis \) organization cookamunga tourist bureau lines 16 article h7v agate berkeley edu , dzkriz ocf berkeley edu \( dennis kriz \) wrote i'm going try something , perhaps many would thought even possible want begin process initiating literary critical study pornography posted alt sex stories , identify major themes motifs present stories posted opening possibility objective moral evaluation material present dennis , i'm astounded n't know interested even study filth alt sex stories provide cheers , kent sandvik newton apple com alink ksand private activities net"
+talk.religion.misc,"anthony landreneau ozonehole com \( anthony landreneau \) subject abortion distribution world organization ozone online operations , inc , dba ozone hole bbs reply anthony landreneau ozonehole com \( anthony landreneau \) lines 21 margoli watson ibm com \( larry margolis \) anthony landreneau ozonehole com lm rape passed , nothing ever take away lm true forcing remain pregnant continues violation lm body another 9 months see unbelievably cruel life violation cruel , killing living solely friend right cold anthony slmr 2 1 's difference orange \? ozone hole bbs private bulletin board service \( 504 \) 891 3142 3 full service nodes usrobotics 16 8k bps 10 gigs 100 , 000 files skydive new orleans ! rime network mail hub 500 usenet newsgroups please route questions inquiries postmaster ozonehole com"
+talk.religion.misc,"kevin rotag mi org \( kevin darcy \) subject 2000 years , say christian morality organization , \? \? \? lines 15 article pww spac at1 59 rice edu pww spacsun rice edu \( peter walker \) writes article 1993apr18 rotag mi org , kevin rotag mi org \( kevin darcy \) wrote , one , considered intentionality primary ontological stuff built perceptions , consciousness , thoughts , etc frank means alone seeing intentionality \( values , puts \) underlying human experience , even called objective experiences , measurements natural world , output des chip others us see intellectual masturbation 'll defer greater firsthand knowledge matters kevin"
+talk.religion.misc,"bil okcforum osrhe edu \( bill conner \) subject 2000 years , say christian morality nntp posting host okcforum osrhe edu organization okcforum unix users group x newsreader tin version 1 1 pl9 lines 54 mind , say science basis values bit reach science basis observable fact 'd say one chooses observe observation interpreted significance 's given depends great deal values observer science human activity , subject potential distortion human activity myth scientists moral influence ethical concern , knowledge whole pure nature biases scientist , nonsense bill one argue objective values \( moral sense \) one must first start demonstrating morality objective considering meaning word objective doubt ever happen , back original question objective morality \? may unfortunate choice words , almost self contradictory objective sense used means something immutable absolute morality describes behavior group people first term inclusive , second specific concept supposedly described may meaning however god described christians \( instance \) , existence apart independent humankind existence outside frame reference \( reality \) declares thing , necessarily since defined omnipotent , claims believed , least omnipotent relative us god intrinsically self defined reality whatever says objective sense god determines standard conduct , standard objective human beings held accountable conformance standard permitted ignore , substitute relative morality mode conduct , giving term morality nebulous , meaningless sense argued pretending misunderstand standard objective conduct required meet standard therefore objectively determined convenient pretend term morality infinitely , n't mean objective standard n't exist morality come mean little cultural norm , preferred conduct decent people , making seem subjective , derived absolute , objective , standard ironically , objective standard perfect accord true nature \( according christianity least \) , yet condemned contrary human , oppressive severe may due bill much amoral standard , like , 's x"