RapidMiner: filtering out n-grams with n>3 - rapidminer

Question

I have about 2 millions of messages (Data Tables).
I'd like to filter out messages containing frequent X-gram, while X>3.
(frequency in % of all messages)
For example:
Message 1 = "1 2 3 4 5"
Message 2 = "1 2 3 4 6"
Message 3 = "1 2 3"
M1 and M2 both have 4-gram 1_2_3_4, so I wand to exclude them, so the result has to leave only M3.

Answer 1

You can use the text processing extension to find n-grams, count how many are longer than three and add that number to the example set to allow subsequent filtering. You can also retain the original data.
Here's an example that you could copy (note that you have to install the text mining extension from the RapidMiner marketplace)
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<process version="6.5.000">
<context>
<input/>
<output/>
<macros/>
</context>
<operator activated="true" class="process" compatibility="6.5.000" expanded="true" name="Process">
<parameter key="logverbosity" value="init"/>
<parameter key="random_seed" value="2001"/>
<parameter key="send_mail" value="never"/>
<parameter key="notification_email" value=""/>
<parameter key="process_duration_for_mail" value="30"/>
<parameter key="encoding" value="SYSTEM"/>
<parameter key="parallelize_main_process" value="false"/>
<process expanded="true">
<operator activated="true" class="generate_data_user_specification" compatibility="6.5.000" expanded="true" height="60" name="Generate Data by User Specification" width="90" x="45" y="75">
<list key="attribute_values">
<parameter key="message" value=""1 2 3 4""/>
</list>
<list key="set_additional_roles"/>
</operator>
<operator activated="true" class="generate_data_user_specification" compatibility="6.5.000" expanded="true" height="60" name="Generate Data by User Specification (2)" width="90" x="45" y="165">
<list key="attribute_values">
<parameter key="message" value=""1 2 3 4 5""/>
</list>
<list key="set_additional_roles"/>
</operator>
<operator activated="true" class="generate_data_user_specification" compatibility="6.5.000" expanded="true" height="60" name="Generate Data by User Specification (3)" width="90" x="45" y="255">
<list key="attribute_values">
<parameter key="message" value=""1 2 3""/>
</list>
<list key="set_additional_roles"/>
</operator>
<operator activated="true" class="append" compatibility="6.5.000" expanded="true" height="112" name="Append" width="90" x="246" y="75">
<parameter key="datamanagement" value="double_array"/>
<parameter key="merge_type" value="all"/>
</operator>
<operator activated="true" class="nominal_to_text" compatibility="6.5.000" expanded="true" height="76" name="Nominal to Text" width="90" x="380" y="75">
<parameter key="attribute_filter_type" value="all"/>
<parameter key="attribute" value=""/>
<parameter key="attributes" value=""/>
<parameter key="use_except_expression" value="false"/>
<parameter key="value_type" value="nominal"/>
<parameter key="use_value_type_exception" value="false"/>
<parameter key="except_value_type" value="file_path"/>
<parameter key="block_type" value="single_value"/>
<parameter key="use_block_type_exception" value="false"/>
<parameter key="except_block_type" value="single_value"/>
<parameter key="invert_selection" value="false"/>
<parameter key="include_special_attributes" value="false"/>
</operator>
<operator activated="true" class="text:process_document_from_data" compatibility="6.5.000" expanded="true" height="76" name="Process Documents from Data" width="90" x="514" y="75">
<parameter key="create_word_vector" value="false"/>
<parameter key="vector_creation" value="Term Occurrences"/>
<parameter key="add_meta_information" value="true"/>
<parameter key="keep_text" value="true"/>
<parameter key="prune_method" value="none"/>
<parameter key="prune_below_percent" value="3.0"/>
<parameter key="prune_above_percent" value="30.0"/>
<parameter key="prune_below_rank" value="0.05"/>
<parameter key="prune_above_rank" value="0.95"/>
<parameter key="datamanagement" value="double_sparse_array"/>
<parameter key="select_attributes_and_weights" value="false"/>
<list key="specify_weights"/>
<parameter key="parallelize_vector_creation" value="false"/>
<process expanded="true">
<operator activated="true" class="multiply" compatibility="6.5.000" expanded="true" height="94" name="Multiply" width="90" x="44" y="30"/>
<operator activated="true" class="text:tokenize" compatibility="6.5.000" expanded="true" height="60" name="Tokenize" width="90" x="179" y="30">
<parameter key="mode" value="regular expression"/>
<parameter key="characters" value=".:"/>
<parameter key="expression" value="\s"/>
<parameter key="language" value="English"/>
<parameter key="max_token_length" value="3"/>
</operator>
<operator activated="true" class="text:generate_n_grams_terms" compatibility="6.5.000" expanded="true" height="60" name="Generate n-Grams (Terms)" width="90" x="179" y="120">
<parameter key="max_length" value="4"/>
</operator>
<operator activated="true" class="text:filter_tokens_by_content" compatibility="6.5.000" expanded="true" height="60" name="Filter Tokens (by Content)" width="90" x="179" y="210">
<parameter key="condition" value="contains match"/>
<parameter key="regular_expression" value="(_.){3,}"/>
<parameter key="case_sensitive" value="false"/>
<parameter key="invert condition" value="false"/>
</operator>
<operator activated="true" class="text:extract_token_number" compatibility="6.5.000" expanded="true" height="60" name="Extract Token Number" width="90" x="179" y="300">
<parameter key="metadata_key" value="numberOfNGramsGT3"/>
<parameter key="condition" value="all"/>
<parameter key="case_sensitive" value="false"/>
<parameter key="invert_condition" value="false"/>
</operator>
<connect from_port="document" to_op="Multiply" to_port="input"/>
<connect from_op="Multiply" from_port="output 1" to_op="Tokenize" to_port="document"/>
<connect from_op="Multiply" from_port="output 2" to_port="document 1"/>
<connect from_op="Tokenize" from_port="document" to_op="Generate n-Grams (Terms)" to_port="document"/>
<connect from_op="Generate n-Grams (Terms)" from_port="document" to_op="Filter Tokens (by Content)" to_port="document"/>
<connect from_op="Filter Tokens (by Content)" from_port="document" to_op="Extract Token Number" to_port="document"/>
<portSpacing port="source_document" spacing="0"/>
<portSpacing port="sink_document 1" spacing="0"/>
<portSpacing port="sink_document 2" spacing="0"/>
</process>
</operator>
<operator activated="true" class="filter_examples" compatibility="6.5.000" expanded="true" height="94" name="Filter Examples" width="90" x="782" y="75">
<parameter key="parameter_expression" value=""/>
<parameter key="condition_class" value="custom_filters"/>
<parameter key="invert_filter" value="false"/>
<list key="filters_list">
<parameter key="filters_entry_key" value="numberOfNGramsGT3.eq.0"/>
</list>
<parameter key="filters_logic_and" value="true"/>
<parameter key="filters_check_metadata" value="true"/>
</operator>
<connect from_op="Generate Data by User Specification" from_port="output" to_op="Append" to_port="example set 1"/>
<connect from_op="Generate Data by User Specification (2)" from_port="output" to_op="Append" to_port="example set 2"/>
<connect from_op="Generate Data by User Specification (3)" from_port="output" to_op="Append" to_port="example set 3"/>
<connect from_op="Append" from_port="merged set" to_op="Nominal to Text" to_port="example set input"/>
<connect from_op="Nominal to Text" from_port="example set output" to_op="Process Documents from Data" to_port="example set"/>
<connect from_op="Process Documents from Data" from_port="example set" to_op="Filter Examples" to_port="example set input"/>
<connect from_op="Filter Examples" from_port="example set output" to_port="result 1"/>
<portSpacing port="source_input 1" spacing="0"/>
<portSpacing port="sink_result 1" spacing="0"/>
<portSpacing port="sink_result 2" spacing="0"/>
</process>
</operator>
</process>