Fix data preparation

src/preparaing_recipe_nlg_dataset.py

@@ -81,6 +81,7 @@ def main():
             cache_dir=data_args.cache_dir
         )
 
+
     def cleaning(text, item_type="ner"):
         # NOTE: DO THE CLEANING LATER
         text = normalizer(text, do_lowercase=True)
@@ -92,9 +93,9 @@ def main():
         ingredients = item_dict["ingredients"]
         steps = item_dict["directions"]
 
-        condition_1 = filter_by_item(ner, 4)
-        condition_2 = filter_by_length(title, 10)
-        condition_3 = filter_by_item(ingredients, 4)
+        condition_1 = filter_by_item(ner, 3)
+        condition_2 = filter_by_length(title, 3)
+        condition_3 = filter_by_item(ingredients, 3)
         condition_4 = filter_by_item(steps, 2)
         condition_5 = filter_by_steps(" ".join(steps))
 
@@ -140,6 +141,10 @@ def main():
                 data_dict.append(item)
 
         data_df = pd.DataFrame(data_dict)
+
+        logger.info(f"Preparation - [before] consists of {len(dataset[subset])} records!")
+        logger.info(f"Preparation - [after]  consists of {len(data_df)} records!")
+
         train, test = train_test_split(data_df, test_size=0.05, random_state=101)
 
         train = train.reset_index(drop=True)

src/run.sh

@@ -52,4 +52,5 @@ python run_ed_recipe_nlg.py \
     --do_train \
     --do_eval \
     --overwrite_output_dir \
-    --predict_with_generate
+    --predict_with_generate \
+    --push_to_hub