AI custom support image column

dtable_events/automations/actions.py

@@ -3494,17 +3494,34 @@ def get_classify_content(self, row_data):
         # Build complete AI classification input content
         content = f'Available options: {target_content}\nOption type: {target_column.get("type")}\nContent to classify: {classify_content}\n'
         return content
+
+    def _collect_image(self, image_list, repo_id):
+        """Collect binary content of the first image from image column"""
+        if not image_list or not repo_id:
+            return None
+
+        try:
+            file_name, download_token = self.get_file_download_info(image_list, repo_id)
+            if not file_name or not download_token:
+                return None
+            return self.get_file_binary_content(file_name, download_token)
+        except Exception as e:
+            auto_rule_logger.warning(f'rule {self.auto_rule.rule_id} collect image failed: {e}')
+            return None
+
     def fill_custom_prompt(self, custom_prompt, row_data):
         """Fill custom_prompt with field values using the same pattern as notifications"""
         if not custom_prompt:
-            return ''
-        
+            return '', []
+
         # Find all field placeholders like {field_name}
         blanks = set(re.findall(r'\{([^{]*?)\}', custom_prompt))
         col_name_dict = {col.get('name'): col for col in self.auto_rule.table_info['columns']}
         column_blanks = [blank for blank in blanks if blank in col_name_dict]
-        
+
         filled_prompt = custom_prompt
+        repo_id = self.config.get('repo_id')
+        images = []
 
         for blank in column_blanks:
             column_value = row_data.get(blank, '')
@@ -3513,13 +3530,19 @@ def fill_custom_prompt(self, custom_prompt, row_data):
             column = col_name_dict.get(blank)
             if column:
                 column_type = column.get('type')
-                formatted_value = self._format_column_value_for_ai(column_value, column_type)
+                if column_type == ColumnTypes.IMAGE and column_value and repo_id:
+                    image_content = self._collect_image(column_value, repo_id)
+                    if image_content:
+                        images.append(image_content)
+                    formatted_value = '[Image provided]'
+                else:
+                    formatted_value = self._format_column_value_for_ai(column_value, column_type)
             else:
                 formatted_value = ''
 
             filled_prompt = filled_prompt.replace('{' + blank + '}', formatted_value)
-        
-        return filled_prompt
+
+        return filled_prompt, images
 
     def summary(self):
         self.fill_summary_field()
@@ -3866,14 +3889,14 @@ def custom(self):
 
         # Process custom_prompt with field replacements
         custom_prompt = self.config.get('custom_prompt', '')
-        filled_prompt = self.fill_custom_prompt(custom_prompt, converted_row)
+        filled_prompt, images = self.fill_custom_prompt(custom_prompt, converted_row)
         filled_prompt = filled_prompt[:AUTO_RULES_AI_CONTENT_MAX_LENGTH]
         if not filled_prompt.strip():
             custom_result = ''
         else:
             try:
                 seatable_ai_api = DTableAIAPI(self.username, self.auto_rule.org_id, self.auto_rule.dtable_uuid, INNER_SEATABLE_AI_SERVER_URL)
-                custom_result = seatable_ai_api.custom(filled_prompt)
+                custom_result = seatable_ai_api.custom(filled_prompt, images=images if images else None)
             except requests.Timeout as timeout_exc:
                 auto_rule_logger.error(f'rule {self.auto_rule.rule_id} ai custom processing timeout {timeout_exc}')
                 self.auto_rule.append_warning({
@@ -5134,6 +5157,7 @@ def do_actions(self, db_session, with_test=False):
                             'config': {
                                 'custom_output_column_key': action_info.get('custom_output_column_key'),
                                 'custom_prompt': action_info.get('custom_prompt'),
+                                'repo_id': action_info.get('repo_id'),
                             }
                         },
                         'invoice_recognition': {

dtable_events/utils/dtable_ai_api.py

@@ -134,7 +134,7 @@ def extract(self, content, extract_fields, extract_prompt):
             logger.error(f"Failed to extract information: {response.text}")
             raise DTableAIAPIError()
 
-    def custom(self, content):
+    def custom(self, content, images=None):
         """Execute custom AI processing with user-defined prompt"""
         if not content or not content.strip():
             return ''
@@ -148,8 +148,13 @@ def custom(self, content):
 
         url = f'{self.seatable_ai_server_url}/api/v1/ai/custom/'
         headers = gen_headers()
-        response = requests.post(url, json=data, headers=headers, timeout=180)
-
+
+        if images:
+            files = [('file', (f'image_{i}.jpg', img, 'image/jpeg')) for i, img in enumerate(images)]
+            response = requests.post(url, data=data, files=files, headers=headers, timeout=180)
+        else:
+            response = requests.post(url, json=data, headers=headers, timeout=180)
+
         if response.status_code == 200:
             result = response.json()
             return result.get('result', '')