Use required information from model field config

.github/workflows/markdown_link_check_config.json

@@ -1,5 +1,8 @@
 {
   "ignorePatterns": [
+    {
+      "pattern": "^https://docs.lucidtech.ai/getting-started/tutorials/"
+    },
     {
       "pattern": "^https://sourcey.com/moxygen"
     },

docs/workflows/transitions/preprocess/preprocess/make_predictions.py

@@ -52,6 +52,16 @@ def get_num_pages(las_client, document_id, max_prediction_pages):
     return min(len(pdf.pages), max_prediction_pages)
 
 
+def add_info_from_field_config(form_config, field_config):
+    try:
+        for key, value in field_config.items():
+            form_config['config']['fields'][key]['required'] = value.get('required', True)
+        return form_config
+    except Exception as e:
+        logging.error(f'Error adding info from field config: {e}')
+        return form_config
+
+
 @las.transition_handler
 def make_predictions(las_client, event):
     document_id = event['documentId']
@@ -84,6 +94,8 @@ def make_predictions(las_client, event):
         form_config = create_form_config_from_model(model_field_config, form_config)
         logging.info(f'\nlabels in fieldConfig does not match form_config. Updated form_config used is: {form_config}')
 
+    form_config = add_info_from_field_config(form_config, model_field_config)
+
     no_empty_prediction_fields = set()
 
     if not (predictions := event.get('predictions')):
@@ -191,9 +203,12 @@ def make_predictions(las_client, event):
                         prediction['confidence'] = 0.0
                     if not above_threshold_or_optional(prediction, field_config):
                         all_above_threshold_or_optional = False
-
-                has_all_required_labels = required_labels(field_config) <= set(map(lambda p: p['label'], top1_preds))
+                _required_labels = required_labels(field_config)
+                _top_1_labels = set(map(lambda p: p['label'], top1_preds))
+                has_all_required_labels = _required_labels <= _top_1_labels
                 needs_validation = not has_all_required_labels or not all_above_threshold_or_optional
+                logging.info(f'required labels: {_required_labels}')
+                logging.info(f"existing labels: {_top_1_labels}")
 
             logging.info(f'All predictions above threshold (or optional): {all_above_threshold_or_optional}')
             logging.info(f'All required labels exist: {has_all_required_labels}')

docs/workflows/transitions/preprocess/preprocess/utils.py

@@ -105,14 +105,15 @@ def patch_and_filter_predictions(predictions, field_config, labels, merge_contin
 
 
 def above_threshold_or_optional(prediction, field_config):
-    label, confidence = prediction['label'], prediction.get('confidence')
+    label, confidence, value = prediction['label'], prediction.get('confidence'), prediction.get('value')
     if label not in field_config:
         return False
 
     threshold = field_config[label]['confidenceLevels']
     is_optional = not field_config[label].get('required', True)
+    valid_optional_prediction = confidence < threshold['low'] or not value
 
-    return (threshold['automated'] <= confidence) or (is_optional and confidence < threshold['low'])
+    return (threshold['automated'] <= confidence) or (is_optional and valid_optional_prediction)
 
 
 def threshold_is_zero_for_all(field_config):

docs/workflows/transitions/preprocess/tests/test_handler.py

@@ -77,7 +77,7 @@ def simple_model_field_config():
     return {
         'total_amount': {},
         'due_date': {},
-        'invoice_id': {},
+        'invoice_id': {'required': False},
         'currency': {},
         'line_items': {
             'type': 'lines',