Utilizing AI for web scraping in Salesforce

				
					global class firecrawlMasterDataSource implements ccai.AIDataSourceInterface {
    global String getExternalData(ccai_AI_Data_Source_c dataSource, String extractedData) {
        try {
            // Parse the extracted data to retrieve website URLs
            System.debug('Extracted Data (Raw Input): ' + extractedData);

            Map<String, Object> inputData = (Map<String, Object>) JSON.deserializeUntyped(extractedData);

            String url1 = (String) inputData.get('CompetitorWebSite1__c');
            String url2 = (String) inputData.get('CompetitorWebSite2__c');

            // Validate URLs
            if (String.isBlank(url1) || String.isBlank(url2)) {
                throw new IllegalArgumentException('One or both competitor websites are missing.');
            }

            // Fetch data from FirecrawlService
            Map<String, Object> combinedData = makeCallouts(url1, url2);

            // Return the combined data as a JSON string
            return JSON.serializePretty(combinedData);
        } catch (Exception e) {
            System.debug('Error: ' + e.getMessage());
            throw new CustomException('Error processing external data: ' + e.getMessage());
        }
    }

    // Helper method for making callouts
    private Map<String, Object> makeCallouts(String url1, String url2) {
        FirecrawlService.FirecrawlResponse response1 = FirecrawlService.fetchWebsiteData(url1);
        FirecrawlService.FirecrawlResponse response2 = FirecrawlService.fetchWebsiteData(url2);

        // Convert responses to Strings and trim to 5000 characters
        String response1Data = response1 != null && response1.data != null 
            ? String.valueOf(response1.data).substring(0, Math.min(String.valueOf(response1.data).length(), 5000)) 
            : null;
        String response2Data = response2 != null && response2.data != null 
            ? String.valueOf(response2.data).substring(0, Math.min(String.valueOf(response2.data).length(), 5000)) 
            : null;

        // Combine the results into a single structure
        Map<String, Object> combinedData = new Map<String, Object>();
        combinedData.put('Website1Data', response1Data);
        combinedData.put('Website2Data', response2Data);

        System.debug('Trimmed Combined Data: ' + JSON.serializePretty(combinedData));
        return combinedData;
    }

    // Define a custom exception to replace ccai.AIDataSourceException
    global class CustomException extends Exception {}
}