tiny cleanup

TobiObeck · Nov 15, 2021 · 5378948 · 5378948
1 parent f283e8d
commit 5378948
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/src/preprocessing/run_preprocessing.py b/src/preprocessing/run_preprocessing.py
@@ -43,15 +43,15 @@ def main():
     if args.tokenize:
         preprocessors.append(Tokenizer(args.tokenize_input, args.tokenize_input + SUFFIX_TOKENIZED))
     if args.other:   
+        preprocessors.append(NonEnglishRemover())
         DROP_COLS = [            
             "id", "conversation_id", "created_at", "timezone", "user_id", "name", "place",
             "replies_count", "retweets_count", "likes_count", "language",
             # "cashtag" only few records have this filled. Might be useless
             # below columns have always the same value for all records
             "retweet", "near", "geo", "source", "user_rt_id", "user_rt", "retweet_id",
             "retweet_date", "translate", "trans_src", 'trans_dest\r']
-
-        preprocessors.append(NonEnglishRemover())
+
         preprocessors.append(ColumnDropper(DROP_COLS))
 
     # call all preprocessing steps