diff --git a/dtable_events/automations/actions.py b/dtable_events/automations/actions.py index 6aebf156..9333b348 100644 --- a/dtable_events/automations/actions.py +++ b/dtable_events/automations/actions.py @@ -3494,17 +3494,34 @@ def get_classify_content(self, row_data): # Build complete AI classification input content content = f'Available options: {target_content}\nOption type: {target_column.get("type")}\nContent to classify: {classify_content}\n' return content + + def _collect_image(self, image_list, repo_id): + """Collect binary content of the first image from image column""" + if not image_list or not repo_id: + return None + + try: + file_name, download_token = self.get_file_download_info(image_list, repo_id) + if not file_name or not download_token: + return None + return self.get_file_binary_content(file_name, download_token) + except Exception as e: + auto_rule_logger.warning(f'rule {self.auto_rule.rule_id} collect image failed: {e}') + return None + def fill_custom_prompt(self, custom_prompt, row_data): """Fill custom_prompt with field values using the same pattern as notifications""" if not custom_prompt: - return '' - + return '', [] + # Find all field placeholders like {field_name} blanks = set(re.findall(r'\{([^{]*?)\}', custom_prompt)) col_name_dict = {col.get('name'): col for col in self.auto_rule.table_info['columns']} column_blanks = [blank for blank in blanks if blank in col_name_dict] - + filled_prompt = custom_prompt + repo_id = self.config.get('repo_id') + images = [] for blank in column_blanks: column_value = row_data.get(blank, '') @@ -3513,13 +3530,19 @@ def fill_custom_prompt(self, custom_prompt, row_data): column = col_name_dict.get(blank) if column: column_type = column.get('type') - formatted_value = self._format_column_value_for_ai(column_value, column_type) + if column_type == ColumnTypes.IMAGE and column_value and repo_id: + image_content = self._collect_image(column_value, repo_id) + if image_content: + images.append(image_content) + formatted_value = '[Image provided]' + else: + formatted_value = self._format_column_value_for_ai(column_value, column_type) else: formatted_value = '' filled_prompt = filled_prompt.replace('{' + blank + '}', formatted_value) - - return filled_prompt + + return filled_prompt, images def summary(self): self.fill_summary_field() @@ -3866,14 +3889,14 @@ def custom(self): # Process custom_prompt with field replacements custom_prompt = self.config.get('custom_prompt', '') - filled_prompt = self.fill_custom_prompt(custom_prompt, converted_row) + filled_prompt, images = self.fill_custom_prompt(custom_prompt, converted_row) filled_prompt = filled_prompt[:AUTO_RULES_AI_CONTENT_MAX_LENGTH] if not filled_prompt.strip(): custom_result = '' else: try: seatable_ai_api = DTableAIAPI(self.username, self.auto_rule.org_id, self.auto_rule.dtable_uuid, INNER_SEATABLE_AI_SERVER_URL) - custom_result = seatable_ai_api.custom(filled_prompt) + custom_result = seatable_ai_api.custom(filled_prompt, images=images if images else None) except requests.Timeout as timeout_exc: auto_rule_logger.error(f'rule {self.auto_rule.rule_id} ai custom processing timeout {timeout_exc}') self.auto_rule.append_warning({ @@ -5134,6 +5157,7 @@ def do_actions(self, db_session, with_test=False): 'config': { 'custom_output_column_key': action_info.get('custom_output_column_key'), 'custom_prompt': action_info.get('custom_prompt'), + 'repo_id': action_info.get('repo_id'), } }, 'invoice_recognition': { diff --git a/dtable_events/utils/dtable_ai_api.py b/dtable_events/utils/dtable_ai_api.py index 4e9531a1..82bff3e9 100644 --- a/dtable_events/utils/dtable_ai_api.py +++ b/dtable_events/utils/dtable_ai_api.py @@ -134,7 +134,7 @@ def extract(self, content, extract_fields, extract_prompt): logger.error(f"Failed to extract information: {response.text}") raise DTableAIAPIError() - def custom(self, content): + def custom(self, content, images=None): """Execute custom AI processing with user-defined prompt""" if not content or not content.strip(): return '' @@ -148,8 +148,13 @@ def custom(self, content): url = f'{self.seatable_ai_server_url}/api/v1/ai/custom/' headers = gen_headers() - response = requests.post(url, json=data, headers=headers, timeout=180) - + + if images: + files = [('file', (f'image_{i}.jpg', img, 'image/jpeg')) for i, img in enumerate(images)] + response = requests.post(url, data=data, files=files, headers=headers, timeout=180) + else: + response = requests.post(url, json=data, headers=headers, timeout=180) + if response.status_code == 200: result = response.json() return result.get('result', '')