Enhance emoji detection by utilizing a predefined emoji list and updating related tests

Author: sundy1994

src/team_comm_tools/features/reddit_tags.py

@@ -1,7 +1,7 @@
 import numpy as np
 import string
 import re
-
+from team_comm_tools.utils.preprocess import EMOJIS
 
 def count_all_caps(text):
     """
@@ -191,6 +191,9 @@ def count_emojis(text):
     Returns:
         int: The number of emojis in the input text.
     """
-    emoji_pattern = r'[:;]-?\)+'
-    emojis = re.findall(emoji_pattern, text)
+    # emoji_pattern = r'[:;]-?\)+'
+    emoji_list = sorted(EMOJIS, key=len, reverse=True)
+    emoji_pattern = "|".join(re.escape(e) for e in emoji_list)
+    compiled_pattern = re.compile(emoji_pattern)
+    emojis = re.findall(compiled_pattern, text)
     return len(emojis)

src/team_comm_tools/utils/preprocess.py

@@ -1,7 +1,16 @@
 import re
 import pandas as pd
-import warnings
-
+# import warnings
+
+EMOJIS = {
+    "(:", "(;", "):", "/:", ":(", ":)", ":/", ";)", # 8 emojis from LIWC 2017
+    ";(", # variants
+    ":-)", ":-(", ":-/", ";-)", # with noses
+    ":D", ":P", ":p", ":-D", ":-P", ":-p", # big grin & tongue out
+    ":O", ":-O", ":o", ":-o", # shock
+    "XD", "xD", "xd", # laughing variants
+    "<3", "</3", # hearts
+}
 
 def preprocess_conversation_columns(df: pd.DataFrame, column_names: dict, grouping_keys: list, 
                                     cumulative_grouping: bool = False, within_task: bool = False) -> pd.DataFrame:
@@ -112,21 +121,19 @@ def preprocess_text(text: str) -> str:
     :return: The processed text containing only alphanumeric characters and spaces in lowercase.
     :rtype: str
     """
-    emojis_to_preserve = {
-        "(:", "(;", "):", "/:", ":(", ":)", ":/", ";)"
-    }
 
     emoji_placeholders = {}
     # Replace each emoji with a unique placeholder
-    for i, emoji in enumerate(emojis_to_preserve):
+    for i, emoji in enumerate(EMOJIS):
         placeholder = f"EMOJI_{i}"
         emoji_placeholders[placeholder] = emoji
         text = text.replace(emoji, placeholder)
 
     # Clean the text by removing unwanted characters, except placeholders
     text = re.sub(r"[^a-zA-Z0-9 EMOJI_]+", '', text)
     # Restore the preserved emojis by replacing placeholders back to original emojis
-    for placeholder, emoji in emoji_placeholders.items():
+    for placeholder in sorted(emoji_placeholders.keys(), key=len, reverse=True):
+        emoji = emoji_placeholders[placeholder]
         text = text.replace(placeholder, emoji)
 
     return text.lower()

tests/data/cleaned_data/test_chat_level.csv

@@ -1332,3 +1332,64 @@ K,1,This is the same text.,positivity_zscore_conversation,
 10_mix,first_person,"belief% sed— forgot{ euismod£ dolor/ couldn't¥ ipsum^ amet— affectation] Lorem> ipsum} drank^ Lorem) sed' forever[ amet/ mailed? euismod< sed? mightve[ dolor[ sed; affect' dolor> sed\ consistently' Lorem| euismod: may: tempor, colon( ipsum# xanax¥ dolor] cruel, amet... think– dolor< tempor, strikingly% euismod< cocks£ ipsum~ insides! ipsum. aren't- Lorem. sandy% adipiscing... consectetur) illuminating+ amet# steadiness= euismod¥ sed( except~ consectetur' tempor) most} adipiscing? here's"" tempor' dolor# understand; sed{ ipsum",first_person_lexical_wordcount,0
 10_mix,nltk_english_stopwords,"belief% sed— forgot{ euismod£ dolor/ couldn't¥ ipsum^ amet— affectation] Lorem> ipsum} drank^ Lorem) sed' forever[ amet/ mailed? euismod< sed? mightve[ dolor[ sed; affect' dolor> sed\ consistently' Lorem| euismod: may: tempor, colon( ipsum# xanax¥ dolor] cruel, amet... think– dolor< tempor, strikingly% euismod< cocks£ ipsum~ insides! ipsum. aren't- Lorem. sandy% adipiscing... consectetur) illuminating+ amet# steadiness= euismod¥ sed( except~ consectetur' tempor) most} adipiscing? here's"" tempor' dolor# understand; sed{ ipsum",nltk_english_stopwords_lexical_wordcount,5
 10_mix,hedge_words,"belief% sed— forgot{ euismod£ dolor/ couldn't¥ ipsum^ amet— affectation] Lorem> ipsum} drank^ Lorem) sed' forever[ amet/ mailed? euismod< sed? mightve[ dolor[ sed; affect' dolor> sed\ consistently' Lorem| euismod: may: tempor, colon( ipsum# xanax¥ dolor] cruel, amet... think– dolor< tempor, strikingly% euismod< cocks£ ipsum~ insides! ipsum. aren't- Lorem. sandy% adipiscing... consectetur) illuminating+ amet# steadiness= euismod¥ sed( except~ consectetur' tempor) most} adipiscing? here's"" tempor' dolor# understand; sed{ ipsum",hedge_words_lexical_wordcount,0
+emoji_test,emoji_user_a,Hello :),num_emoji,1
+emoji_test,emoji_user_b,Nice to see you :-)),num_emoji,1
+emoji_test,emoji_user_a,Hey there ;-) How are you? :),num_emoji,2
+emoji_test,emoji_user_b,No emoji here.,num_emoji,0
+emoji_test,emoji_user_a,;) :) :-))),num_emoji,3
+emoji_test,emoji_user_b,Mixed emotions: :-( ;( :( ,num_emoji,3
+emoji_test,emoji_user_a,Too many smiles :) :) :),num_emoji,3
+emoji_test,emoji_user_b,Tricky one: :-)-),num_emoji,1
+emoji_test,emoji_user_a,Extra characters :-))abc:-),num_emoji,2
+emoji_test,emoji_user_b,Combo ;-):-);-),num_emoji,3
+emoji_test,emoji_user_a,Sad day :(,num_emoji,1
+emoji_test,emoji_user_b,Confused face :/,num_emoji,1
+emoji_test,emoji_user_a,Double smiles :) :),num_emoji,2
+emoji_test,emoji_user_b,This is fine :D,num_emoji,1
+emoji_test,emoji_user_a,Edge case :-):-):-D,num_emoji,3
+emoji_test,emoji_user_b,Broken smile :-))abc:-),num_emoji,2
+emoji_test,emoji_user_a,Mixed (: ;),num_emoji,2
+emoji_test,emoji_user_b,Only symbols $%&!,num_emoji,0
+emoji_test,emoji_user_a,Nested :)hello:),num_emoji,2
+emoji_test,emoji_user_b,Sad then happy :( :),num_emoji,2
+emoji_test,emoji_user_a,No spaces:;-):),num_emoji,2
+emoji_test,emoji_user_b,Triple threat :):):),num_emoji,3
+emoji_test,emoji_user_a,Reverse smile ): (:,num_emoji,2
+emoji_test,emoji_user_b,"Hey! I'm doing great :-), thanks for asking. How about you? ;)",num_emoji,2
+emoji_test,emoji_user_a,Ugh... today was rough :( Everything just kept going wrong :-(,num_emoji,2
+emoji_test,emoji_user_b,LOL! That was hilarious :D:D:D!! Can't stop laughing!,num_emoji,3
+emoji_test,emoji_user_a,Okay... so he said 'I'll be there :)' — and then never showed up.,num_emoji,1
+emoji_test,emoji_user_b,"Meeting recaD:
+- Client happy :)
+- Budget approved ;)
+- Launch on track :-D",num_emoji,3
+emoji_test,emoji_user_a,"Meeting recap:
+- Client happy :)
+- Budget approved ;)
+- Launch on track :-D",num_emoji,3
+emoji_test,emoji_user_b,Mixed signals: first he smiled :) then frowned :( then smiled again :)),num_emoji,3
+emoji_test,emoji_user_a,Edge-case test —;-):-(;-))... any of these match?,num_emoji,3
+emoji_test,emoji_user_b,"Email thread:
+> Sure, that works for me :)
+> Thanks! :-)
+> See you then ;)",num_emoji,3
+emoji_test,emoji_user_a,Total chaos!!! :(( :(( :(( — but at least coffee :-),num_emoji,4
+emoji_test,emoji_user_b,"He walked in and said, 'Good morning :-)' — like nothing happened!",num_emoji,1
+emoji_test,emoji_user_a,Haha :D that was funny!,num_emoji,1
+emoji_test,emoji_user_b,You got me :-P,num_emoji,1
+emoji_test,emoji_user_a,Just kidding :p :P,num_emoji,2
+emoji_test,emoji_user_b,All the reactions: :D :-D :P :-P,num_emoji,4
+emoji_test,emoji_user_a,Lowercase alert :-p :p,num_emoji,2
+emoji_test,emoji_user_b,No way! :O,num_emoji,1
+emoji_test,emoji_user_a,You're kidding :-O,num_emoji,1
+emoji_test,emoji_user_b,Mixed shock faces: :o :-o,num_emoji,2
+emoji_test,emoji_user_a,Wow :O :o :-O :-o,num_emoji,4
+emoji_test,emoji_user_b,OMG XD that cracked me up,num_emoji,1
+emoji_test,emoji_user_a,xD lol,num_emoji,1
+emoji_test,emoji_user_b,xd XD xD,num_emoji,3
+emoji_test,emoji_user_a,Try this combo: xdxd,num_emoji,2
+emoji_test,emoji_user_b,Real XDPro was here,num_emoji,1
+emoji_test,emoji_user_a,I love this <3,num_emoji,1
+emoji_test,emoji_user_b,Broken heart </3,num_emoji,1
+emoji_test,emoji_user_a,Love and heartbreak: <3 </3,num_emoji,2
+emoji_test,emoji_user_b,Wow<3cool</3,num_emoji,2